drivers/net/mvpp2/mrvl_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017-2021 Marvell International Ltd.
   3  * Copyright(c) 2017-2021 Semihalf.
   4  * All rights reserved.
   5  */
   6
   7 #include <rte_string_fns.h>
   8 #include <ethdev_driver.h>
   9 #include <rte_kvargs.h>
  10 #include <rte_log.h>
  11 #include <rte_malloc.h>
  12 #include <rte_bus_vdev.h>
  13
  14 #include <fcntl.h>
  15 #include <linux/ethtool.h>
  16 #include <linux/sockios.h>
  17 #include <net/if.h>
  18 #include <net/if_arp.h>
  19 #include <sys/ioctl.h>
  20 #include <sys/socket.h>
  21 #include <sys/stat.h>
  22 #include <sys/types.h>
  23
  24 #include <rte_mvep_common.h>
  25 #include "mrvl_ethdev.h"
  26 #include "mrvl_qos.h"
  27 #include "mrvl_flow.h"
  28 #include "mrvl_mtr.h"
  29 #include "mrvl_tm.h"
  30
  31 /* bitmask with reserved hifs */
  32 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  33 /* bitmask with reserved bpools */
  34 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  35 /* bitmask with reserved kernel RSS tables */
  36 #define MRVL_MUSDK_RSS_RESERVED 0x0F
  37 /* maximum number of available hifs */
  38 #define MRVL_MUSDK_HIFS_MAX 9
  39
  40 /* prefetch shift */
  41 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  42
  43 /* TCAM has 25 entries reserved for uc/mc filter entries
  44  * + 1 for primary mac address
  45  */
  46 #define MRVL_MAC_ADDRS_MAX (1 + 25)
  47 #define MRVL_MATCH_LEN 16
  48 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  49 /* Maximum allowable packet size */
  50 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  51
  52 #define MRVL_IFACE_NAME_ARG "iface"
  53 #define MRVL_CFG_ARG "cfg"
  54
  55 #define MRVL_BURST_SIZE 64
  56
  57 #define MRVL_ARP_LENGTH 28
  58
  59 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  60 #define MRVL_COOKIE_HIGH_ADDR_MASK 0xffffff0000000000
  61
  62 /** Port Rx offload capabilities */
  63 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
  64                           DEV_RX_OFFLOAD_JUMBO_FRAME | \
  65                           DEV_RX_OFFLOAD_CHECKSUM)
  66
  67 /** Port Tx offloads capabilities */
  68 #define MRVL_TX_OFFLOAD_CHECKSUM (DEV_TX_OFFLOAD_IPV4_CKSUM | \
  69                                   DEV_TX_OFFLOAD_UDP_CKSUM  | \
  70                                   DEV_TX_OFFLOAD_TCP_CKSUM)
  71 #define MRVL_TX_OFFLOADS (MRVL_TX_OFFLOAD_CHECKSUM | \
  72                           DEV_TX_OFFLOAD_MULTI_SEGS)
  73
  74 #define MRVL_TX_PKT_OFFLOADS (PKT_TX_IP_CKSUM | \
  75                               PKT_TX_TCP_CKSUM | \
  76                               PKT_TX_UDP_CKSUM)
  77
  78 static const char * const valid_args[] = {
  79         MRVL_IFACE_NAME_ARG,
  80         MRVL_CFG_ARG,
  81         NULL
  82 };
  83
  84 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
  85 static struct pp2_hif *hifs[RTE_MAX_LCORE];
  86 static int used_bpools[PP2_NUM_PKT_PROC] = {
  87         [0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED
  88 };
  89
  90 static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
  91 static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
  92 static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
  93 static int dummy_pool_id[PP2_NUM_PKT_PROC];
  94 struct pp2_bpool *dummy_pool[PP2_NUM_PKT_PROC] = {0};
  95
  96 struct mrvl_ifnames {
  97         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
  98         int idx;
  99 };
 100
 101 /*
 102  * To use buffer harvesting based on loopback port shadow queue structure
 103  * was introduced for buffers information bookkeeping.
 104  *
 105  * Before sending the packet, related buffer information (pp2_buff_inf) is
 106  * stored in shadow queue. After packet is transmitted no longer used
 107  * packet buffer is released back to it's original hardware pool,
 108  * on condition it originated from interface.
 109  * In case it  was generated by application itself i.e: mbuf->port field is
 110  * 0xff then its released to software mempool.
 111  */
 112 struct mrvl_shadow_txq {
 113         int head;           /* write index - used when sending buffers */
 114         int tail;           /* read index - used when releasing buffers */
 115         u16 size;           /* queue occupied size */
 116         u16 num_to_release; /* number of descriptors sent, that can be
 117                              * released
 118                              */
 119         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 120 };
 121
 122 struct mrvl_rxq {
 123         struct mrvl_priv *priv;
 124         struct rte_mempool *mp;
 125         int queue_id;
 126         int port_id;
 127         int cksum_enabled;
 128         uint64_t bytes_recv;
 129         uint64_t drop_mac;
 130 };
 131
 132 struct mrvl_txq {
 133         struct mrvl_priv *priv;
 134         int queue_id;
 135         int port_id;
 136         uint64_t bytes_sent;
 137         struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE];
 138         int tx_deferred_start;
 139 };
 140
 141 static int mrvl_lcore_first;
 142 static int mrvl_lcore_last;
 143 static int mrvl_dev_num;
 144
 145 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
 146 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
 147                         struct pp2_hif *hif, unsigned int core_id,
 148                         struct mrvl_shadow_txq *sq, int qid, int force);
 149
 150 static uint16_t mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
 151                                   uint16_t nb_pkts);
 152 static uint16_t mrvl_tx_sg_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
 153                                      uint16_t nb_pkts);
 154 static int rte_pmd_mrvl_remove(struct rte_vdev_device *vdev);
 155 static void mrvl_deinit_pp2(void);
 156 static void mrvl_deinit_hifs(void);
 157
 158 static int
 159 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
 160                   uint32_t index, uint32_t vmdq __rte_unused);
 161 static int
 162 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr);
 163 static int
 164 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
 165 static int mrvl_promiscuous_enable(struct rte_eth_dev *dev);
 166 static int mrvl_allmulticast_enable(struct rte_eth_dev *dev);
 167 static int
 168 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf);
 169
 170 #define MRVL_XSTATS_TBL_ENTRY(name) { \
 171         #name, offsetof(struct pp2_ppio_statistics, name),      \
 172         sizeof(((struct pp2_ppio_statistics *)0)->name)         \
 173 }
 174
 175 /* Table with xstats data */
 176 static struct {
 177         const char *name;
 178         unsigned int offset;
 179         unsigned int size;
 180 } mrvl_xstats_tbl[] = {
 181         MRVL_XSTATS_TBL_ENTRY(rx_bytes),
 182         MRVL_XSTATS_TBL_ENTRY(rx_packets),
 183         MRVL_XSTATS_TBL_ENTRY(rx_unicast_packets),
 184         MRVL_XSTATS_TBL_ENTRY(rx_errors),
 185         MRVL_XSTATS_TBL_ENTRY(rx_fullq_dropped),
 186         MRVL_XSTATS_TBL_ENTRY(rx_bm_dropped),
 187         MRVL_XSTATS_TBL_ENTRY(rx_early_dropped),
 188         MRVL_XSTATS_TBL_ENTRY(rx_fifo_dropped),
 189         MRVL_XSTATS_TBL_ENTRY(rx_cls_dropped),
 190         MRVL_XSTATS_TBL_ENTRY(tx_bytes),
 191         MRVL_XSTATS_TBL_ENTRY(tx_packets),
 192         MRVL_XSTATS_TBL_ENTRY(tx_unicast_packets),
 193         MRVL_XSTATS_TBL_ENTRY(tx_errors)
 194 };
 195
 196 static inline int
 197 mrvl_reserve_bit(int *bitmap, int max)
 198 {
 199         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 200
 201         if (n >= max)
 202                 return -1;
 203
 204         *bitmap |= 1 << n;
 205
 206         return n;
 207 }
 208
 209 static int
 210 mrvl_pp2_fixup_init(void)
 211 {
 212         struct pp2_bpool_params bpool_params;
 213         char                    name[15];
 214         int                     err, i;
 215
 216         memset(dummy_pool, 0, sizeof(dummy_pool));
 217         for (i = 0; i < pp2_get_num_inst(); i++) {
 218                 dummy_pool_id[i] = mrvl_reserve_bit(&used_bpools[i],
 219                                              PP2_BPOOL_NUM_POOLS);
 220                 if (dummy_pool_id[i] < 0) {
 221                         MRVL_LOG(ERR, "Can't find free pool\n");
 222                         return -1;
 223                 }
 224
 225                 memset(name, 0, sizeof(name));
 226                 snprintf(name, sizeof(name), "pool-%d:%d", i, dummy_pool_id[i]);
 227                 memset(&bpool_params, 0, sizeof(bpool_params));
 228                 bpool_params.match = name;
 229                 bpool_params.buff_len = MRVL_PKT_OFFS;
 230                 bpool_params.dummy_short_pool = 1;
 231                 err = pp2_bpool_init(&bpool_params, &dummy_pool[i]);
 232                 if (err != 0 || !dummy_pool[i]) {
 233                         MRVL_LOG(ERR, "BPool init failed!\n");
 234                         used_bpools[i] &= ~(1 << dummy_pool_id[i]);
 235                         return -1;
 236                 }
 237         }
 238
 239         return 0;
 240 }
 241
 242 /**
 243  * Initialize packet processor.
 244  *
 245  * @return
 246  *   0 on success, negative error value otherwise.
 247  */
 248 static int
 249 mrvl_init_pp2(void)
 250 {
 251         struct pp2_init_params  init_params;
 252         int                     err;
 253
 254         memset(&init_params, 0, sizeof(init_params));
 255         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
 256         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
 257         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
 258         if (mrvl_cfg && mrvl_cfg->pp2_cfg.prs_udfs.num_udfs)
 259                 memcpy(&init_params.prs_udfs, &mrvl_cfg->pp2_cfg.prs_udfs,
 260                        sizeof(struct pp2_parse_udfs));
 261         err = pp2_init(&init_params);
 262         if (err != 0) {
 263                 MRVL_LOG(ERR, "PP2 init failed");
 264                 return -1;
 265         }
 266
 267         err = mrvl_pp2_fixup_init();
 268         if (err != 0) {
 269                 MRVL_LOG(ERR, "PP2 fixup init failed");
 270                 return -1;
 271         }
 272
 273         return 0;
 274 }
 275
 276 static void
 277 mrvl_pp2_fixup_deinit(void)
 278 {
 279         int i;
 280
 281         for (i = 0; i < PP2_NUM_PKT_PROC; i++) {
 282                 if (!dummy_pool[i])
 283                         continue;
 284                 pp2_bpool_deinit(dummy_pool[i]);
 285                 used_bpools[i] &= ~(1 << dummy_pool_id[i]);
 286         }
 287 }
 288
 289 /**
 290  * Deinitialize packet processor.
 291  *
 292  * @return
 293  *   0 on success, negative error value otherwise.
 294  */
 295 static void
 296 mrvl_deinit_pp2(void)
 297 {
 298         mrvl_pp2_fixup_deinit();
 299         pp2_deinit();
 300 }
 301
 302 static inline void
 303 mrvl_fill_shadowq(struct mrvl_shadow_txq *sq, struct rte_mbuf *buf)
 304 {
 305         sq->ent[sq->head].buff.cookie = (uint64_t)buf;
 306         sq->ent[sq->head].buff.addr = buf ?
 307                 rte_mbuf_data_iova_default(buf) : 0;
 308
 309         sq->ent[sq->head].bpool =
 310                 (unlikely(!buf || buf->port >= RTE_MAX_ETHPORTS ||
 311                  buf->refcnt > 1)) ? NULL :
 312                  mrvl_port_to_bpool_lookup[buf->port];
 313
 314         sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
 315         sq->size++;
 316 }
 317
 318 /**
 319  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
 320  */
 321 static void
 322 mrvl_deinit_hifs(void)
 323 {
 324         int i;
 325
 326         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
 327                 if (hifs[i])
 328                         pp2_hif_deinit(hifs[i]);
 329         }
 330         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
 331         memset(hifs, 0, sizeof(hifs));
 332 }
 333
 334 static inline void
 335 mrvl_fill_desc(struct pp2_ppio_desc *desc, struct rte_mbuf *buf)
 336 {
 337         pp2_ppio_outq_desc_reset(desc);
 338         pp2_ppio_outq_desc_set_phys_addr(desc, rte_pktmbuf_iova(buf));
 339         pp2_ppio_outq_desc_set_pkt_offset(desc, 0);
 340         pp2_ppio_outq_desc_set_pkt_len(desc, rte_pktmbuf_data_len(buf));
 341 }
 342
 343 static inline int
 344 mrvl_get_bpool_size(int pp2_id, int pool_id)
 345 {
 346         int i;
 347         int size = 0;
 348
 349         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 350                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 351
 352         return size;
 353 }
 354
 355 static int
 356 mrvl_init_hif(int core_id)
 357 {
 358         struct pp2_hif_params params;
 359         char match[MRVL_MATCH_LEN];
 360         int ret;
 361
 362         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
 363         if (ret < 0) {
 364                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 365                 return ret;
 366         }
 367
 368         snprintf(match, sizeof(match), "hif-%d", ret);
 369         memset(&params, 0, sizeof(params));
 370         params.match = match;
 371         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
 372         ret = pp2_hif_init(&params, &hifs[core_id]);
 373         if (ret) {
 374                 MRVL_LOG(ERR, "Failed to initialize hif %d", core_id);
 375                 return ret;
 376         }
 377
 378         return 0;
 379 }
 380
 381 static inline struct pp2_hif*
 382 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
 383 {
 384         int ret;
 385
 386         if (likely(hifs[core_id] != NULL))
 387                 return hifs[core_id];
 388
 389         rte_spinlock_lock(&priv->lock);
 390
 391         ret = mrvl_init_hif(core_id);
 392         if (ret < 0) {
 393                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 394                 goto out;
 395         }
 396
 397         if (core_id < mrvl_lcore_first)
 398                 mrvl_lcore_first = core_id;
 399
 400         if (core_id > mrvl_lcore_last)
 401                 mrvl_lcore_last = core_id;
 402 out:
 403         rte_spinlock_unlock(&priv->lock);
 404
 405         return hifs[core_id];
 406 }
 407
 408 /**
 409  * Set tx burst function according to offload flag
 410  *
 411  * @param dev
 412  *   Pointer to Ethernet device structure.
 413  */
 414 static void
 415 mrvl_set_tx_function(struct rte_eth_dev *dev)
 416 {
 417         struct mrvl_priv *priv = dev->data->dev_private;
 418
 419         /* Use a simple Tx queue (no offloads, no multi segs) if possible */
 420         if (priv->multiseg) {
 421                 RTE_LOG(INFO, PMD, "Using multi-segment tx callback\n");
 422                 dev->tx_pkt_burst = mrvl_tx_sg_pkt_burst;
 423         } else {
 424                 RTE_LOG(INFO, PMD, "Using single-segment tx callback\n");
 425                 dev->tx_pkt_burst = mrvl_tx_pkt_burst;
 426         }
 427 }
 428
 429 /**
 430  * Configure rss based on dpdk rss configuration.
 431  *
 432  * @param priv
 433  *   Pointer to private structure.
 434  * @param rss_conf
 435  *   Pointer to RSS configuration.
 436  *
 437  * @return
 438  *   0 on success, negative error value otherwise.
 439  */
 440 static int
 441 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 442 {
 443         if (rss_conf->rss_key)
 444                 MRVL_LOG(WARNING, "Changing hash key is not supported");
 445
 446         if (rss_conf->rss_hf == 0) {
 447                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 448         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 449                 priv->ppio_params.inqs_params.hash_type =
 450                         PP2_PPIO_HASH_T_2_TUPLE;
 451         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 452                 priv->ppio_params.inqs_params.hash_type =
 453                         PP2_PPIO_HASH_T_5_TUPLE;
 454                 priv->rss_hf_tcp = 1;
 455         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 456                 priv->ppio_params.inqs_params.hash_type =
 457                         PP2_PPIO_HASH_T_5_TUPLE;
 458                 priv->rss_hf_tcp = 0;
 459         } else {
 460                 return -EINVAL;
 461         }
 462
 463         return 0;
 464 }
 465
 466 /**
 467  * Ethernet device configuration.
 468  *
 469  * Prepare the driver for a given number of TX and RX queues and
 470  * configure RSS.
 471  *
 472  * @param dev
 473  *   Pointer to Ethernet device structure.
 474  *
 475  * @return
 476  *   0 on success, negative error value otherwise.
 477  */
 478 static int
 479 mrvl_dev_configure(struct rte_eth_dev *dev)
 480 {
 481         struct mrvl_priv *priv = dev->data->dev_private;
 482         int ret;
 483
 484         if (priv->ppio) {
 485                 MRVL_LOG(INFO, "Device reconfiguration is not supported");
 486                 return -EINVAL;
 487         }
 488
 489         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 490             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 491                 MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
 492                         dev->data->dev_conf.rxmode.mq_mode);
 493                 return -EINVAL;
 494         }
 495
 496         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 497                 MRVL_LOG(INFO, "Split headers not supported");
 498                 return -EINVAL;
 499         }
 500
 501         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
 502                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 503                                  MRVL_PP2_ETH_HDRS_LEN;
 504                 if (dev->data->mtu > priv->max_mtu) {
 505                         MRVL_LOG(ERR, "inherit MTU %u from max_rx_pkt_len %u is larger than max_mtu %u\n",
 506                                  dev->data->mtu,
 507                                  dev->data->dev_conf.rxmode.max_rx_pkt_len,
 508                                  priv->max_mtu);
 509                         return -EINVAL;
 510                 }
 511         }
 512
 513         if (dev->data->dev_conf.txmode.offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
 514                 priv->multiseg = 1;
 515
 516         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 517                                   dev->data->nb_rx_queues);
 518         if (ret < 0)
 519                 return ret;
 520
 521         ret = mrvl_configure_txqs(priv, dev->data->port_id,
 522                                   dev->data->nb_tx_queues);
 523         if (ret < 0)
 524                 return ret;
 525
 526         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 527         priv->ppio_params.maintain_stats = 1;
 528         priv->nb_rx_queues = dev->data->nb_rx_queues;
 529
 530         ret = mrvl_tm_init(dev);
 531         if (ret < 0)
 532                 return ret;
 533
 534         if (dev->data->nb_rx_queues == 1 &&
 535             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 536                 MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
 537                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 538
 539                 return 0;
 540         }
 541
 542         return mrvl_configure_rss(priv,
 543                         &dev->data->dev_conf.rx_adv_conf.rss_conf);
 544 }
 545
 546 /**
 547  * DPDK callback to change the MTU.
 548  *
 549  * Setting the MTU affects hardware MRU (packets larger than the MRU
 550  * will be dropped).
 551  *
 552  * @param dev
 553  *   Pointer to Ethernet device structure.
 554  * @param mtu
 555  *   New MTU.
 556  *
 557  * @return
 558  *   0 on success, negative error value otherwise.
 559  */
 560 static int
 561 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 562 {
 563         struct mrvl_priv *priv = dev->data->dev_private;
 564         uint16_t mru;
 565         uint16_t mbuf_data_size = 0; /* SW buffer size */
 566         int ret;
 567
 568         mru = MRVL_PP2_MTU_TO_MRU(mtu);
 569         /*
 570          * min_rx_buf_size is equal to mbuf data size
 571          * if pmd didn't set it differently
 572          */
 573         mbuf_data_size = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
 574         /* Prevent PMD from:
 575          * - setting mru greater than the mbuf size resulting in
 576          * hw and sw buffer size mismatch
 577          * - setting mtu that requires the support of scattered packets
 578          * when this feature has not been enabled/supported so far
 579          * (TODO check scattered_rx flag here once scattered RX is supported).
 580          */
 581         if (mru - RTE_ETHER_CRC_LEN + MRVL_PKT_OFFS > mbuf_data_size) {
 582                 mru = mbuf_data_size + RTE_ETHER_CRC_LEN - MRVL_PKT_OFFS;
 583                 mtu = MRVL_PP2_MRU_TO_MTU(mru);
 584                 MRVL_LOG(WARNING, "MTU too big, max MTU possible limitted "
 585                         "by current mbuf size: %u. Set MTU to %u, MRU to %u",
 586                         mbuf_data_size, mtu, mru);
 587         }
 588
 589         if (mtu < RTE_ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX) {
 590                 MRVL_LOG(ERR, "Invalid MTU [%u] or MRU [%u]", mtu, mru);
 591                 return -EINVAL;
 592         }
 593
 594         dev->data->mtu = mtu;
 595         dev->data->dev_conf.rxmode.max_rx_pkt_len = mru - MV_MH_SIZE;
 596
 597         if (!priv->ppio)
 598                 return 0;
 599
 600         ret = pp2_ppio_set_mru(priv->ppio, mru);
 601         if (ret) {
 602                 MRVL_LOG(ERR, "Failed to change MRU");
 603                 return ret;
 604         }
 605
 606         ret = pp2_ppio_set_mtu(priv->ppio, mtu);
 607         if (ret) {
 608                 MRVL_LOG(ERR, "Failed to change MTU");
 609                 return ret;
 610         }
 611
 612         return 0;
 613 }
 614
 615 /**
 616  * DPDK callback to bring the link up.
 617  *
 618  * @param dev
 619  *   Pointer to Ethernet device structure.
 620  *
 621  * @return
 622  *   0 on success, negative error value otherwise.
 623  */
 624 static int
 625 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 626 {
 627         struct mrvl_priv *priv = dev->data->dev_private;
 628         int ret;
 629
 630         if (!priv->ppio) {
 631                 dev->data->dev_link.link_status = ETH_LINK_UP;
 632                 return 0;
 633         }
 634
 635         ret = pp2_ppio_enable(priv->ppio);
 636         if (ret)
 637                 return ret;
 638
 639         /*
 640          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 641          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 642          * PP2_TRAFFIC_INGRESS_EGRESS.
 643          *
 644          * Set mtu to default DPDK value here.
 645          */
 646         ret = mrvl_mtu_set(dev, dev->data->mtu);
 647         if (ret) {
 648                 pp2_ppio_disable(priv->ppio);
 649                 return ret;
 650         }
 651
 652         dev->data->dev_link.link_status = ETH_LINK_UP;
 653         return 0;
 654 }
 655
 656 /**
 657  * DPDK callback to bring the link down.
 658  *
 659  * @param dev
 660  *   Pointer to Ethernet device structure.
 661  *
 662  * @return
 663  *   0 on success, negative error value otherwise.
 664  */
 665 static int
 666 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 667 {
 668         struct mrvl_priv *priv = dev->data->dev_private;
 669         int ret;
 670
 671         if (!priv->ppio) {
 672                 dev->data->dev_link.link_status = ETH_LINK_DOWN;
 673                 return 0;
 674         }
 675         ret = pp2_ppio_disable(priv->ppio);
 676         if (ret)
 677                 return ret;
 678
 679         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 680         return 0;
 681 }
 682
 683 /**
 684  * DPDK callback to start tx queue.
 685  *
 686  * @param dev
 687  *   Pointer to Ethernet device structure.
 688  * @param queue_id
 689  *   Transmit queue index.
 690  *
 691  * @return
 692  *   0 on success, negative error value otherwise.
 693  */
 694 static int
 695 mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
 696 {
 697         struct mrvl_priv *priv = dev->data->dev_private;
 698         int ret;
 699
 700         if (!priv)
 701                 return -EPERM;
 702
 703         /* passing 1 enables given tx queue */
 704         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1);
 705         if (ret) {
 706                 MRVL_LOG(ERR, "Failed to start txq %d", queue_id);
 707                 return ret;
 708         }
 709
 710         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 711
 712         return 0;
 713 }
 714
 715 /**
 716  * DPDK callback to stop tx queue.
 717  *
 718  * @param dev
 719  *   Pointer to Ethernet device structure.
 720  * @param queue_id
 721  *   Transmit queue index.
 722  *
 723  * @return
 724  *   0 on success, negative error value otherwise.
 725  */
 726 static int
 727 mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
 728 {
 729         struct mrvl_priv *priv = dev->data->dev_private;
 730         int ret;
 731
 732         if (!priv->ppio)
 733                 return -EPERM;
 734
 735         /* passing 0 disables given tx queue */
 736         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0);
 737         if (ret) {
 738                 MRVL_LOG(ERR, "Failed to stop txq %d", queue_id);
 739                 return ret;
 740         }
 741
 742         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 743
 744         return 0;
 745 }
 746
 747 /**
 748  * Populate VLAN Filter configuration.
 749  *
 750  * @param dev
 751  *   Pointer to Ethernet device structure.
 752  * @param on
 753  *   Toggle filter.
 754  *
 755  * @return
 756  *   0 on success, negative error value otherwise.
 757  */
 758 static int mrvl_populate_vlan_table(struct rte_eth_dev *dev, int on)
 759 {
 760         uint32_t j;
 761         int ret;
 762         struct rte_vlan_filter_conf *vfc;
 763
 764         vfc = &dev->data->vlan_filter_conf;
 765         for (j = 0; j < RTE_DIM(vfc->ids); j++) {
 766                 uint64_t vlan;
 767                 uint64_t vbit;
 768                 uint64_t ids = vfc->ids[j];
 769
 770                 if (ids == 0)
 771                         continue;
 772
 773                 while (ids) {
 774                         vlan = 64 * j;
 775                         /* count trailing zeroes */
 776                         vbit = ~ids & (ids - 1);
 777                         /* clear least significant bit set */
 778                         ids ^= (ids ^ (ids - 1)) ^ vbit;
 779                         for (; vbit; vlan++)
 780                                 vbit >>= 1;
 781                         ret = mrvl_vlan_filter_set(dev, vlan, on);
 782                         if (ret) {
 783                                 MRVL_LOG(ERR, "Failed to setup VLAN filter\n");
 784                                 return ret;
 785                         }
 786                 }
 787         }
 788
 789         return 0;
 790 }
 791
 792 /**
 793  * DPDK callback to start the device.
 794  *
 795  * @param dev
 796  *   Pointer to Ethernet device structure.
 797  *
 798  * @return
 799  *   0 on success, negative errno value on failure.
 800  */
 801 static int
 802 mrvl_dev_start(struct rte_eth_dev *dev)
 803 {
 804         struct mrvl_priv *priv = dev->data->dev_private;
 805         char match[MRVL_MATCH_LEN];
 806         int ret = 0, i, def_init_size;
 807         struct rte_ether_addr *mac_addr;
 808
 809         if (priv->ppio)
 810                 return mrvl_dev_set_link_up(dev);
 811
 812         snprintf(match, sizeof(match), "ppio-%d:%d",
 813                  priv->pp_id, priv->ppio_id);
 814         priv->ppio_params.match = match;
 815         priv->ppio_params.eth_start_hdr = PP2_PPIO_HDR_ETH;
 816         if (mrvl_cfg)
 817                 priv->ppio_params.eth_start_hdr =
 818                         mrvl_cfg->port[dev->data->port_id].eth_start_hdr;
 819
 820         /*
 821          * Calculate the minimum bpool size for refill feature as follows:
 822          * 2 default burst sizes multiply by number of rx queues.
 823          * If the bpool size will be below this value, new buffers will
 824          * be added to the pool.
 825          */
 826         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 827
 828         /* In case initial bpool size configured in queues setup is
 829          * smaller than minimum size add more buffers
 830          */
 831         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
 832         if (priv->bpool_init_size < def_init_size) {
 833                 int buffs_to_add = def_init_size - priv->bpool_init_size;
 834
 835                 priv->bpool_init_size += buffs_to_add;
 836                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
 837                 if (ret)
 838                         MRVL_LOG(ERR, "Failed to add buffers to bpool");
 839         }
 840
 841         /*
 842          * Calculate the maximum bpool size for refill feature as follows:
 843          * maximum number of descriptors in rx queue multiply by number
 844          * of rx queues plus minimum bpool size.
 845          * In case the bpool size will exceed this value, superfluous buffers
 846          * will be removed
 847          */
 848         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
 849                                 priv->bpool_min_size;
 850
 851         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 852         if (ret) {
 853                 MRVL_LOG(ERR, "Failed to init ppio");
 854                 return ret;
 855         }
 856
 857         /*
 858          * In case there are some some stale uc/mc mac addresses flush them
 859          * here. It cannot be done during mrvl_dev_close() as port information
 860          * is already gone at that point (due to pp2_ppio_deinit() in
 861          * mrvl_dev_stop()).
 862          */
 863         if (!priv->uc_mc_flushed) {
 864                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 865                 if (ret) {
 866                         MRVL_LOG(ERR,
 867                                 "Failed to flush uc/mc filter list");
 868                         goto out;
 869                 }
 870                 priv->uc_mc_flushed = 1;
 871         }
 872
 873         ret = mrvl_mtu_set(dev, dev->data->mtu);
 874         if (ret)
 875                 MRVL_LOG(ERR, "Failed to set MTU to %d", dev->data->mtu);
 876
 877         if (!rte_is_zero_ether_addr(&dev->data->mac_addrs[0]))
 878                 mrvl_mac_addr_set(dev, &dev->data->mac_addrs[0]);
 879
 880         for (i = 1; i < MRVL_MAC_ADDRS_MAX; i++) {
 881                 mac_addr = &dev->data->mac_addrs[i];
 882
 883                 /* skip zero address */
 884                 if (rte_is_zero_ether_addr(mac_addr))
 885                         continue;
 886
 887                 mrvl_mac_addr_add(dev, mac_addr, i, 0);
 888         }
 889
 890         if (dev->data->all_multicast == 1)
 891                 mrvl_allmulticast_enable(dev);
 892
 893         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_VLAN_FILTER) {
 894                 ret = mrvl_populate_vlan_table(dev, 1);
 895                 if (ret) {
 896                         MRVL_LOG(ERR, "Failed to populate VLAN table");
 897                         goto out;
 898                 }
 899         }
 900
 901         /* For default QoS config, don't start classifier. */
 902         if (mrvl_cfg  &&
 903             mrvl_cfg->port[dev->data->port_id].use_global_defaults == 0) {
 904                 ret = mrvl_start_qos_mapping(priv);
 905                 if (ret) {
 906                         MRVL_LOG(ERR, "Failed to setup QoS mapping");
 907                         goto out;
 908                 }
 909         }
 910
 911         ret = pp2_ppio_set_loopback(priv->ppio, dev->data->dev_conf.lpbk_mode);
 912         if (ret) {
 913                 MRVL_LOG(ERR, "Failed to set loopback");
 914                 goto out;
 915         }
 916
 917         if (dev->data->promiscuous == 1)
 918                 mrvl_promiscuous_enable(dev);
 919
 920         if (priv->flow_ctrl) {
 921                 ret = mrvl_flow_ctrl_set(dev, &priv->fc_conf);
 922                 if (ret) {
 923                         MRVL_LOG(ERR, "Failed to configure flow control");
 924                         goto out;
 925                 }
 926                 priv->flow_ctrl = 0;
 927         }
 928
 929         if (dev->data->dev_link.link_status == ETH_LINK_UP) {
 930                 ret = mrvl_dev_set_link_up(dev);
 931                 if (ret) {
 932                         MRVL_LOG(ERR, "Failed to set link up");
 933                         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 934                         goto out;
 935                 }
 936         }
 937
 938         /* start tx queues */
 939         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 940                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 941
 942                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
 943
 944                 if (!txq->tx_deferred_start)
 945                         continue;
 946
 947                 /*
 948                  * All txqs are started by default. Stop them
 949                  * so that tx_deferred_start works as expected.
 950                  */
 951                 ret = mrvl_tx_queue_stop(dev, i);
 952                 if (ret)
 953                         goto out;
 954         }
 955
 956         mrvl_flow_init(dev);
 957         mrvl_mtr_init(dev);
 958         mrvl_set_tx_function(dev);
 959
 960         return 0;
 961 out:
 962         MRVL_LOG(ERR, "Failed to start device");
 963         pp2_ppio_deinit(priv->ppio);
 964         return ret;
 965 }
 966
 967 /**
 968  * Flush receive queues.
 969  *
 970  * @param dev
 971  *   Pointer to Ethernet device structure.
 972  */
 973 static void
 974 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 975 {
 976         int i;
 977
 978         MRVL_LOG(INFO, "Flushing rx queues");
 979         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 980                 int ret, num;
 981
 982                 do {
 983                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 984                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 985
 986                         num = MRVL_PP2_RXD_MAX;
 987                         ret = pp2_ppio_recv(q->priv->ppio,
 988                                             q->priv->rxq_map[q->queue_id].tc,
 989                                             q->priv->rxq_map[q->queue_id].inq,
 990                                             descs, (uint16_t *)&num);
 991                 } while (ret == 0 && num);
 992         }
 993 }
 994
 995 /**
 996  * Flush transmit shadow queues.
 997  *
 998  * @param dev
 999  *   Pointer to Ethernet device structure.
1000  */
1001 static void
1002 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
1003 {
1004         int i, j;
1005         struct mrvl_txq *txq;
1006
1007         MRVL_LOG(INFO, "Flushing tx shadow queues");
1008         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1009                 txq = (struct mrvl_txq *)dev->data->tx_queues[i];
1010
1011                 for (j = 0; j < RTE_MAX_LCORE; j++) {
1012                         struct mrvl_shadow_txq *sq;
1013
1014                         if (!hifs[j])
1015                                 continue;
1016
1017                         sq = &txq->shadow_txqs[j];
1018                         mrvl_free_sent_buffers(txq->priv->ppio,
1019                                 hifs[j], j, sq, txq->queue_id, 1);
1020                         while (sq->tail != sq->head) {
1021                                 uint64_t addr = cookie_addr_high |
1022                                         sq->ent[sq->tail].buff.cookie;
1023                                 rte_pktmbuf_free(
1024                                         (struct rte_mbuf *)addr);
1025                                 sq->tail = (sq->tail + 1) &
1026                                             MRVL_PP2_TX_SHADOWQ_MASK;
1027                         }
1028                         memset(sq, 0, sizeof(*sq));
1029                 }
1030         }
1031 }
1032
1033 /**
1034  * Flush hardware bpool (buffer-pool).
1035  *
1036  * @param dev
1037  *   Pointer to Ethernet device structure.
1038  */
1039 static void
1040 mrvl_flush_bpool(struct rte_eth_dev *dev)
1041 {
1042         struct mrvl_priv *priv = dev->data->dev_private;
1043         struct pp2_hif *hif;
1044         uint32_t num;
1045         int ret;
1046         unsigned int core_id = rte_lcore_id();
1047
1048         if (core_id == LCORE_ID_ANY)
1049                 core_id = rte_get_main_lcore();
1050
1051         hif = mrvl_get_hif(priv, core_id);
1052
1053         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
1054         if (ret) {
1055                 MRVL_LOG(ERR, "Failed to get bpool buffers number");
1056                 return;
1057         }
1058
1059         while (num--) {
1060                 struct pp2_buff_inf inf;
1061                 uint64_t addr;
1062
1063                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
1064                 if (ret)
1065                         break;
1066
1067                 addr = cookie_addr_high | inf.cookie;
1068                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1069         }
1070 }
1071
1072 /**
1073  * DPDK callback to stop the device.
1074  *
1075  * @param dev
1076  *   Pointer to Ethernet device structure.
1077  */
1078 static int
1079 mrvl_dev_stop(struct rte_eth_dev *dev)
1080 {
1081         return mrvl_dev_set_link_down(dev);
1082 }
1083
1084 /**
1085  * DPDK callback to close the device.
1086  *
1087  * @param dev
1088  *   Pointer to Ethernet device structure.
1089  */
1090 static int
1091 mrvl_dev_close(struct rte_eth_dev *dev)
1092 {
1093         struct mrvl_priv *priv = dev->data->dev_private;
1094         size_t i;
1095
1096         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1097                 return 0;
1098
1099         mrvl_flush_rx_queues(dev);
1100         mrvl_flush_tx_shadow_queues(dev);
1101         mrvl_flow_deinit(dev);
1102         mrvl_mtr_deinit(dev);
1103
1104         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
1105                 struct pp2_ppio_tc_params *tc_params =
1106                         &priv->ppio_params.inqs_params.tcs_params[i];
1107
1108                 if (tc_params->inqs_params) {
1109                         rte_free(tc_params->inqs_params);
1110                         tc_params->inqs_params = NULL;
1111                 }
1112         }
1113
1114         if (priv->cls_tbl) {
1115                 pp2_cls_tbl_deinit(priv->cls_tbl);
1116                 priv->cls_tbl = NULL;
1117         }
1118
1119         if (priv->qos_tbl) {
1120                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
1121                 priv->qos_tbl = NULL;
1122         }
1123
1124         mrvl_flush_bpool(dev);
1125         mrvl_tm_deinit(dev);
1126
1127         if (priv->ppio) {
1128                 pp2_ppio_deinit(priv->ppio);
1129                 priv->ppio = NULL;
1130         }
1131
1132         /* policer must be released after ppio deinitialization */
1133         if (priv->default_policer) {
1134                 pp2_cls_plcr_deinit(priv->default_policer);
1135                 priv->default_policer = NULL;
1136         }
1137
1138
1139         if (priv->bpool) {
1140                 pp2_bpool_deinit(priv->bpool);
1141                 used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
1142                 priv->bpool = NULL;
1143         }
1144
1145         mrvl_dev_num--;
1146
1147         if (mrvl_dev_num == 0) {
1148                 MRVL_LOG(INFO, "Perform MUSDK deinit");
1149                 mrvl_deinit_hifs();
1150                 mrvl_deinit_pp2();
1151                 rte_mvep_deinit(MVEP_MOD_T_PP2);
1152         }
1153
1154         return 0;
1155 }
1156
1157 /**
1158  * DPDK callback to retrieve physical link information.
1159  *
1160  * @param dev
1161  *   Pointer to Ethernet device structure.
1162  * @param wait_to_complete
1163  *   Wait for request completion (ignored).
1164  *
1165  * @return
1166  *   0 on success, negative error value otherwise.
1167  */
1168 static int
1169 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
1170 {
1171         /*
1172          * TODO
1173          * once MUSDK provides necessary API use it here
1174          */
1175         struct mrvl_priv *priv = dev->data->dev_private;
1176         struct ethtool_cmd edata;
1177         struct ifreq req;
1178         int ret, fd, link_up;
1179
1180         if (!priv->ppio)
1181                 return -EPERM;
1182
1183         edata.cmd = ETHTOOL_GSET;
1184
1185         strcpy(req.ifr_name, dev->data->name);
1186         req.ifr_data = (void *)&edata;
1187
1188         fd = socket(AF_INET, SOCK_DGRAM, 0);
1189         if (fd == -1)
1190                 return -EFAULT;
1191
1192         ret = ioctl(fd, SIOCETHTOOL, &req);
1193         if (ret == -1) {
1194                 close(fd);
1195                 return -EFAULT;
1196         }
1197
1198         close(fd);
1199
1200         switch (ethtool_cmd_speed(&edata)) {
1201         case SPEED_10:
1202                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
1203                 break;
1204         case SPEED_100:
1205                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
1206                 break;
1207         case SPEED_1000:
1208                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
1209                 break;
1210         case SPEED_2500:
1211                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_2_5G;
1212                 break;
1213         case SPEED_10000:
1214                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
1215                 break;
1216         default:
1217                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
1218         }
1219
1220         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
1221                                                          ETH_LINK_HALF_DUPLEX;
1222         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
1223                                                            ETH_LINK_FIXED;
1224         pp2_ppio_get_link_state(priv->ppio, &link_up);
1225         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
1226
1227         return 0;
1228 }
1229
1230 /**
1231  * DPDK callback to enable promiscuous mode.
1232  *
1233  * @param dev
1234  *   Pointer to Ethernet device structure.
1235  *
1236  * @return
1237  *   0 on success, negative error value otherwise.
1238  */
1239 static int
1240 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
1241 {
1242         struct mrvl_priv *priv = dev->data->dev_private;
1243         int ret;
1244
1245         if (priv->isolated)
1246                 return -ENOTSUP;
1247
1248         if (!priv->ppio)
1249                 return 0;
1250
1251         ret = pp2_ppio_set_promisc(priv->ppio, 1);
1252         if (ret) {
1253                 MRVL_LOG(ERR, "Failed to enable promiscuous mode");
1254                 return -EAGAIN;
1255         }
1256
1257         return 0;
1258 }
1259
1260 /**
1261  * DPDK callback to enable allmulti mode.
1262  *
1263  * @param dev
1264  *   Pointer to Ethernet device structure.
1265  *
1266  * @return
1267  *   0 on success, negative error value otherwise.
1268  */
1269 static int
1270 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
1271 {
1272         struct mrvl_priv *priv = dev->data->dev_private;
1273         int ret;
1274
1275         if (priv->isolated)
1276                 return -ENOTSUP;
1277
1278         if (!priv->ppio)
1279                 return 0;
1280
1281         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
1282         if (ret) {
1283                 MRVL_LOG(ERR, "Failed enable all-multicast mode");
1284                 return -EAGAIN;
1285         }
1286
1287         return 0;
1288 }
1289
1290 /**
1291  * DPDK callback to disable promiscuous mode.
1292  *
1293  * @param dev
1294  *   Pointer to Ethernet device structure.
1295  *
1296  * @return
1297  *   0 on success, negative error value otherwise.
1298  */
1299 static int
1300 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
1301 {
1302         struct mrvl_priv *priv = dev->data->dev_private;
1303         int ret;
1304
1305         if (priv->isolated)
1306                 return -ENOTSUP;
1307
1308         if (!priv->ppio)
1309                 return 0;
1310
1311         ret = pp2_ppio_set_promisc(priv->ppio, 0);
1312         if (ret) {
1313                 MRVL_LOG(ERR, "Failed to disable promiscuous mode");
1314                 return -EAGAIN;
1315         }
1316
1317         return 0;
1318 }
1319
1320 /**
1321  * DPDK callback to disable allmulticast mode.
1322  *
1323  * @param dev
1324  *   Pointer to Ethernet device structure.
1325  *
1326  * @return
1327  *   0 on success, negative error value otherwise.
1328  */
1329 static int
1330 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
1331 {
1332         struct mrvl_priv *priv = dev->data->dev_private;
1333         int ret;
1334
1335         if (priv->isolated)
1336                 return -ENOTSUP;
1337
1338         if (!priv->ppio)
1339                 return 0;
1340
1341         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
1342         if (ret) {
1343                 MRVL_LOG(ERR, "Failed to disable all-multicast mode");
1344                 return -EAGAIN;
1345         }
1346
1347         return 0;
1348 }
1349
1350 /**
1351  * DPDK callback to remove a MAC address.
1352  *
1353  * @param dev
1354  *   Pointer to Ethernet device structure.
1355  * @param index
1356  *   MAC address index.
1357  */
1358 static void
1359 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1360 {
1361         struct mrvl_priv *priv = dev->data->dev_private;
1362         char buf[RTE_ETHER_ADDR_FMT_SIZE];
1363         int ret;
1364
1365         if (priv->isolated)
1366                 return;
1367
1368         if (!priv->ppio)
1369                 return;
1370
1371         ret = pp2_ppio_remove_mac_addr(priv->ppio,
1372                                        dev->data->mac_addrs[index].addr_bytes);
1373         if (ret) {
1374                 rte_ether_format_addr(buf, sizeof(buf),
1375                                   &dev->data->mac_addrs[index]);
1376                 MRVL_LOG(ERR, "Failed to remove mac %s", buf);
1377         }
1378 }
1379
1380 /**
1381  * DPDK callback to add a MAC address.
1382  *
1383  * @param dev
1384  *   Pointer to Ethernet device structure.
1385  * @param mac_addr
1386  *   MAC address to register.
1387  * @param index
1388  *   MAC address index.
1389  * @param vmdq
1390  *   VMDq pool index to associate address with (unused).
1391  *
1392  * @return
1393  *   0 on success, negative error value otherwise.
1394  */
1395 static int
1396 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1397                   uint32_t index, uint32_t vmdq __rte_unused)
1398 {
1399         struct mrvl_priv *priv = dev->data->dev_private;
1400         char buf[RTE_ETHER_ADDR_FMT_SIZE];
1401         int ret;
1402
1403         if (priv->isolated)
1404                 return -ENOTSUP;
1405
1406         if (!priv->ppio)
1407                 return 0;
1408
1409         if (index == 0)
1410                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
1411                 return -1;
1412
1413         /*
1414          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
1415          * parameter uc_filter_max. Maximum number of mc addresses is then
1416          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
1417          * 21 respectively.
1418          *
1419          * If more than uc_filter_max uc addresses were added to filter list
1420          * then NIC will switch to promiscuous mode automatically.
1421          *
1422          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
1423          * were added to filter list then NIC will switch to all-multicast mode
1424          * automatically.
1425          */
1426         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
1427         if (ret) {
1428                 rte_ether_format_addr(buf, sizeof(buf), mac_addr);
1429                 MRVL_LOG(ERR, "Failed to add mac %s", buf);
1430                 return -1;
1431         }
1432
1433         return 0;
1434 }
1435
1436 /**
1437  * DPDK callback to set the primary MAC address.
1438  *
1439  * @param dev
1440  *   Pointer to Ethernet device structure.
1441  * @param mac_addr
1442  *   MAC address to register.
1443  *
1444  * @return
1445  *   0 on success, negative error value otherwise.
1446  */
1447 static int
1448 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
1449 {
1450         struct mrvl_priv *priv = dev->data->dev_private;
1451         int ret;
1452
1453         if (priv->isolated)
1454                 return -ENOTSUP;
1455
1456         if (!priv->ppio)
1457                 return 0;
1458
1459         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
1460         if (ret) {
1461                 char buf[RTE_ETHER_ADDR_FMT_SIZE];
1462                 rte_ether_format_addr(buf, sizeof(buf), mac_addr);
1463                 MRVL_LOG(ERR, "Failed to set mac to %s", buf);
1464         }
1465
1466         return ret;
1467 }
1468
1469 /**
1470  * DPDK callback to get device statistics.
1471  *
1472  * @param dev
1473  *   Pointer to Ethernet device structure.
1474  * @param stats
1475  *   Stats structure output buffer.
1476  *
1477  * @return
1478  *   0 on success, negative error value otherwise.
1479  */
1480 static int
1481 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1482 {
1483         struct mrvl_priv *priv = dev->data->dev_private;
1484         struct pp2_ppio_statistics ppio_stats;
1485         uint64_t drop_mac = 0;
1486         unsigned int i, idx, ret;
1487
1488         if (!priv->ppio)
1489                 return -EPERM;
1490
1491         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1492                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1493                 struct pp2_ppio_inq_statistics rx_stats;
1494
1495                 if (!rxq)
1496                         continue;
1497
1498                 idx = rxq->queue_id;
1499                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1500                         MRVL_LOG(ERR,
1501                                 "rx queue %d stats out of range (0 - %d)",
1502                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1503                         continue;
1504                 }
1505
1506                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
1507                                                   priv->rxq_map[idx].tc,
1508                                                   priv->rxq_map[idx].inq,
1509                                                   &rx_stats, 0);
1510                 if (unlikely(ret)) {
1511                         MRVL_LOG(ERR,
1512                                 "Failed to update rx queue %d stats", idx);
1513                         break;
1514                 }
1515
1516                 stats->q_ibytes[idx] = rxq->bytes_recv;
1517                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
1518                 stats->q_errors[idx] = rx_stats.drop_early +
1519                                        rx_stats.drop_fullq +
1520                                        rx_stats.drop_bm +
1521                                        rxq->drop_mac;
1522                 stats->ibytes += rxq->bytes_recv;
1523                 drop_mac += rxq->drop_mac;
1524         }
1525
1526         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1527                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1528                 struct pp2_ppio_outq_statistics tx_stats;
1529
1530                 if (!txq)
1531                         continue;
1532
1533                 idx = txq->queue_id;
1534                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1535                         MRVL_LOG(ERR,
1536                                 "tx queue %d stats out of range (0 - %d)",
1537                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1538                 }
1539
1540                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1541                                                    &tx_stats, 0);
1542                 if (unlikely(ret)) {
1543                         MRVL_LOG(ERR,
1544                                 "Failed to update tx queue %d stats", idx);
1545                         break;
1546                 }
1547
1548                 stats->q_opackets[idx] = tx_stats.deq_desc;
1549                 stats->q_obytes[idx] = txq->bytes_sent;
1550                 stats->obytes += txq->bytes_sent;
1551         }
1552
1553         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1554         if (unlikely(ret)) {
1555                 MRVL_LOG(ERR, "Failed to update port statistics");
1556                 return ret;
1557         }
1558
1559         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1560         stats->opackets += ppio_stats.tx_packets;
1561         stats->imissed += ppio_stats.rx_fullq_dropped +
1562                           ppio_stats.rx_bm_dropped +
1563                           ppio_stats.rx_early_dropped +
1564                           ppio_stats.rx_fifo_dropped +
1565                           ppio_stats.rx_cls_dropped;
1566         stats->ierrors = drop_mac;
1567
1568         return 0;
1569 }
1570
1571 /**
1572  * DPDK callback to clear device statistics.
1573  *
1574  * @param dev
1575  *   Pointer to Ethernet device structure.
1576  *
1577  * @return
1578  *   0 on success, negative error value otherwise.
1579  */
1580 static int
1581 mrvl_stats_reset(struct rte_eth_dev *dev)
1582 {
1583         struct mrvl_priv *priv = dev->data->dev_private;
1584         int i;
1585
1586         if (!priv->ppio)
1587                 return 0;
1588
1589         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1590                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1591
1592                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1593                                             priv->rxq_map[i].inq, NULL, 1);
1594                 rxq->bytes_recv = 0;
1595                 rxq->drop_mac = 0;
1596         }
1597
1598         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1599                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1600
1601                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1602                 txq->bytes_sent = 0;
1603         }
1604
1605         return pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1606 }
1607
1608 /**
1609  * DPDK callback to get extended statistics.
1610  *
1611  * @param dev
1612  *   Pointer to Ethernet device structure.
1613  * @param stats
1614  *   Pointer to xstats table.
1615  * @param n
1616  *   Number of entries in xstats table.
1617  * @return
1618  *   Negative value on error, number of read xstats otherwise.
1619  */
1620 static int
1621 mrvl_xstats_get(struct rte_eth_dev *dev,
1622                 struct rte_eth_xstat *stats, unsigned int n)
1623 {
1624         struct mrvl_priv *priv = dev->data->dev_private;
1625         struct pp2_ppio_statistics ppio_stats;
1626         unsigned int i;
1627
1628         if (!stats)
1629                 return 0;
1630
1631         pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1632         for (i = 0; i < n && i < RTE_DIM(mrvl_xstats_tbl); i++) {
1633                 uint64_t val;
1634
1635                 if (mrvl_xstats_tbl[i].size == sizeof(uint32_t))
1636                         val = *(uint32_t *)((uint8_t *)&ppio_stats +
1637                                             mrvl_xstats_tbl[i].offset);
1638                 else if (mrvl_xstats_tbl[i].size == sizeof(uint64_t))
1639                         val = *(uint64_t *)((uint8_t *)&ppio_stats +
1640                                             mrvl_xstats_tbl[i].offset);
1641                 else
1642                         return -EINVAL;
1643
1644                 stats[i].id = i;
1645                 stats[i].value = val;
1646         }
1647
1648         return n;
1649 }
1650
1651 /**
1652  * DPDK callback to reset extended statistics.
1653  *
1654  * @param dev
1655  *   Pointer to Ethernet device structure.
1656  *
1657  * @return
1658  *   0 on success, negative error value otherwise.
1659  */
1660 static int
1661 mrvl_xstats_reset(struct rte_eth_dev *dev)
1662 {
1663         return mrvl_stats_reset(dev);
1664 }
1665
1666 /**
1667  * DPDK callback to get extended statistics names.
1668  *
1669  * @param dev (unused)
1670  *   Pointer to Ethernet device structure.
1671  * @param xstats_names
1672  *   Pointer to xstats names table.
1673  * @param size
1674  *   Size of the xstats names table.
1675  * @return
1676  *   Number of read names.
1677  */
1678 static int
1679 mrvl_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
1680                       struct rte_eth_xstat_name *xstats_names,
1681                       unsigned int size)
1682 {
1683         unsigned int i;
1684
1685         if (!xstats_names)
1686                 return RTE_DIM(mrvl_xstats_tbl);
1687
1688         for (i = 0; i < size && i < RTE_DIM(mrvl_xstats_tbl); i++)
1689                 strlcpy(xstats_names[i].name, mrvl_xstats_tbl[i].name,
1690                         RTE_ETH_XSTATS_NAME_SIZE);
1691
1692         return size;
1693 }
1694
1695 /**
1696  * DPDK callback to get information about the device.
1697  *
1698  * @param dev
1699  *   Pointer to Ethernet device structure (unused).
1700  * @param info
1701  *   Info structure output buffer.
1702  */
1703 static int
1704 mrvl_dev_infos_get(struct rte_eth_dev *dev,
1705                    struct rte_eth_dev_info *info)
1706 {
1707         struct mrvl_priv *priv = dev->data->dev_private;
1708
1709         info->speed_capa = ETH_LINK_SPEED_10M |
1710                            ETH_LINK_SPEED_100M |
1711                            ETH_LINK_SPEED_1G |
1712                            ETH_LINK_SPEED_2_5G |
1713                            ETH_LINK_SPEED_10G;
1714
1715         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1716         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1717         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1718
1719         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1720         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1721         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1722
1723         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1724         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1725         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1726
1727         info->rx_offload_capa = MRVL_RX_OFFLOADS;
1728         info->rx_queue_offload_capa = MRVL_RX_OFFLOADS;
1729
1730         info->tx_offload_capa = MRVL_TX_OFFLOADS;
1731         info->tx_queue_offload_capa = MRVL_TX_OFFLOADS;
1732
1733         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1734                                        ETH_RSS_NONFRAG_IPV4_TCP |
1735                                        ETH_RSS_NONFRAG_IPV4_UDP;
1736
1737         /* By default packets are dropped if no descriptors are available */
1738         info->default_rxconf.rx_drop_en = 1;
1739
1740         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1741         info->max_mtu = priv->max_mtu;
1742
1743         return 0;
1744 }
1745
1746 /**
1747  * Return supported packet types.
1748  *
1749  * @param dev
1750  *   Pointer to Ethernet device structure (unused).
1751  *
1752  * @return
1753  *   Const pointer to the table with supported packet types.
1754  */
1755 static const uint32_t *
1756 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1757 {
1758         static const uint32_t ptypes[] = {
1759                 RTE_PTYPE_L2_ETHER,
1760                 RTE_PTYPE_L2_ETHER_VLAN,
1761                 RTE_PTYPE_L2_ETHER_QINQ,
1762                 RTE_PTYPE_L3_IPV4,
1763                 RTE_PTYPE_L3_IPV4_EXT,
1764                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1765                 RTE_PTYPE_L3_IPV6,
1766                 RTE_PTYPE_L3_IPV6_EXT,
1767                 RTE_PTYPE_L2_ETHER_ARP,
1768                 RTE_PTYPE_L4_TCP,
1769                 RTE_PTYPE_L4_UDP
1770         };
1771
1772         return ptypes;
1773 }
1774
1775 /**
1776  * DPDK callback to get information about specific receive queue.
1777  *
1778  * @param dev
1779  *   Pointer to Ethernet device structure.
1780  * @param rx_queue_id
1781  *   Receive queue index.
1782  * @param qinfo
1783  *   Receive queue information structure.
1784  */
1785 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1786                               struct rte_eth_rxq_info *qinfo)
1787 {
1788         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1789         struct mrvl_priv *priv = dev->data->dev_private;
1790         int inq = priv->rxq_map[rx_queue_id].inq;
1791         int tc = priv->rxq_map[rx_queue_id].tc;
1792         struct pp2_ppio_tc_params *tc_params =
1793                 &priv->ppio_params.inqs_params.tcs_params[tc];
1794
1795         qinfo->mp = q->mp;
1796         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1797 }
1798
1799 /**
1800  * DPDK callback to get information about specific transmit queue.
1801  *
1802  * @param dev
1803  *   Pointer to Ethernet device structure.
1804  * @param tx_queue_id
1805  *   Transmit queue index.
1806  * @param qinfo
1807  *   Transmit queue information structure.
1808  */
1809 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1810                               struct rte_eth_txq_info *qinfo)
1811 {
1812         struct mrvl_priv *priv = dev->data->dev_private;
1813         struct mrvl_txq *txq = dev->data->tx_queues[tx_queue_id];
1814
1815         qinfo->nb_desc =
1816                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1817         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1818 }
1819
1820 /**
1821  * DPDK callback to Configure a VLAN filter.
1822  *
1823  * @param dev
1824  *   Pointer to Ethernet device structure.
1825  * @param vlan_id
1826  *   VLAN ID to filter.
1827  * @param on
1828  *   Toggle filter.
1829  *
1830  * @return
1831  *   0 on success, negative error value otherwise.
1832  */
1833 static int
1834 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1835 {
1836         struct mrvl_priv *priv = dev->data->dev_private;
1837
1838         if (priv->isolated)
1839                 return -ENOTSUP;
1840
1841         if (!priv->ppio)
1842                 return 0;
1843
1844         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1845                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1846 }
1847
1848 /**
1849  * DPDK callback to Configure VLAN offload.
1850  *
1851  * @param dev
1852  *   Pointer to Ethernet device structure.
1853  * @param mask
1854  *   VLAN offload mask.
1855  *
1856  * @return
1857  *   0 on success, negative error value otherwise.
1858  */
1859 static int mrvl_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1860 {
1861         uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
1862         int ret;
1863
1864         if (mask & ETH_VLAN_STRIP_MASK)
1865                 MRVL_LOG(ERR, "VLAN stripping is not supported\n");
1866
1867         if (mask & ETH_VLAN_FILTER_MASK) {
1868                 if (rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
1869                         ret = mrvl_populate_vlan_table(dev, 1);
1870                 else
1871                         ret = mrvl_populate_vlan_table(dev, 0);
1872
1873                 if (ret)
1874                         return ret;
1875         }
1876
1877         if (mask & ETH_VLAN_EXTEND_MASK)
1878                 MRVL_LOG(ERR, "Extend VLAN not supported\n");
1879
1880         return 0;
1881 }
1882
1883 /**
1884  * Release buffers to hardware bpool (buffer-pool)
1885  *
1886  * @param rxq
1887  *   Receive queue pointer.
1888  * @param num
1889  *   Number of buffers to release to bpool.
1890  *
1891  * @return
1892  *   0 on success, negative error value otherwise.
1893  */
1894 static int
1895 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1896 {
1897         struct buff_release_entry entries[num];
1898         struct rte_mbuf *mbufs[num];
1899         int i, ret;
1900         unsigned int core_id;
1901         struct pp2_hif *hif;
1902         struct pp2_bpool *bpool;
1903
1904         core_id = rte_lcore_id();
1905         if (core_id == LCORE_ID_ANY)
1906                 core_id = rte_get_main_lcore();
1907
1908         hif = mrvl_get_hif(rxq->priv, core_id);
1909         if (!hif)
1910                 return -1;
1911
1912         bpool = rxq->priv->bpool;
1913
1914         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1915         if (ret)
1916                 return ret;
1917
1918         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1919                 cookie_addr_high =
1920                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1921
1922         for (i = 0; i < num; i++) {
1923                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1924                         != cookie_addr_high) {
1925                         MRVL_LOG(ERR,
1926                                 "mbuf virtual addr high is out of range "
1927                                 "0x%x instead of 0x%x\n",
1928                                 (uint32_t)((uint64_t)mbufs[i] >> 32),
1929                                 (uint32_t)(cookie_addr_high >> 32));
1930                         goto out;
1931                 }
1932
1933                 entries[i].buff.addr =
1934                         rte_mbuf_data_iova_default(mbufs[i]);
1935                 entries[i].buff.cookie = (uintptr_t)mbufs[i];
1936                 entries[i].bpool = bpool;
1937         }
1938
1939         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1940         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1941
1942         if (i != num)
1943                 goto out;
1944
1945         return 0;
1946 out:
1947         for (; i < num; i++)
1948                 rte_pktmbuf_free(mbufs[i]);
1949
1950         return -1;
1951 }
1952
1953 /**
1954  * DPDK callback to configure the receive queue.
1955  *
1956  * @param dev
1957  *   Pointer to Ethernet device structure.
1958  * @param idx
1959  *   RX queue index.
1960  * @param desc
1961  *   Number of descriptors to configure in queue.
1962  * @param socket
1963  *   NUMA socket on which memory must be allocated.
1964  * @param conf
1965  *   Thresholds parameters.
1966  * @param mp
1967  *   Memory pool for buffer allocations.
1968  *
1969  * @return
1970  *   0 on success, negative error value otherwise.
1971  */
1972 static int
1973 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1974                     unsigned int socket,
1975                     const struct rte_eth_rxconf *conf,
1976                     struct rte_mempool *mp)
1977 {
1978         struct mrvl_priv *priv = dev->data->dev_private;
1979         struct mrvl_rxq *rxq;
1980         uint32_t frame_size, buf_size = rte_pktmbuf_data_room_size(mp);
1981         uint32_t max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1982         int ret, tc, inq;
1983         uint64_t offloads;
1984
1985         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1986
1987         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1988                 /*
1989                  * Unknown TC mapping, mapping will not have a correct queue.
1990                  */
1991                 MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu",
1992                         idx, priv->ppio_id);
1993                 return -EFAULT;
1994         }
1995
1996         frame_size = buf_size - RTE_PKTMBUF_HEADROOM -
1997                      MRVL_PKT_EFFEC_OFFS + RTE_ETHER_CRC_LEN;
1998         if (frame_size < max_rx_pkt_len) {
1999                 MRVL_LOG(WARNING,
2000                         "Mbuf size must be increased to %u bytes to hold up "
2001                         "to %u bytes of data.",
2002                         buf_size + max_rx_pkt_len - frame_size,
2003                         max_rx_pkt_len);
2004                 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
2005                 MRVL_LOG(INFO, "Setting max rx pkt len to %u",
2006                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
2007         }
2008
2009         if (dev->data->rx_queues[idx]) {
2010                 rte_free(dev->data->rx_queues[idx]);
2011                 dev->data->rx_queues[idx] = NULL;
2012         }
2013
2014         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
2015         if (!rxq)
2016                 return -ENOMEM;
2017
2018         rxq->priv = priv;
2019         rxq->mp = mp;
2020         rxq->cksum_enabled = offloads & DEV_RX_OFFLOAD_IPV4_CKSUM;
2021         rxq->queue_id = idx;
2022         rxq->port_id = dev->data->port_id;
2023         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
2024
2025         tc = priv->rxq_map[rxq->queue_id].tc,
2026         inq = priv->rxq_map[rxq->queue_id].inq;
2027         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
2028                 desc;
2029
2030         ret = mrvl_fill_bpool(rxq, desc);
2031         if (ret) {
2032                 rte_free(rxq);
2033                 return ret;
2034         }
2035
2036         priv->bpool_init_size += desc;
2037
2038         dev->data->rx_queues[idx] = rxq;
2039
2040         return 0;
2041 }
2042
2043 /**
2044  * DPDK callback to release the receive queue.
2045  *
2046  * @param rxq
2047  *   Generic receive queue pointer.
2048  */
2049 static void
2050 mrvl_rx_queue_release(void *rxq)
2051 {
2052         struct mrvl_rxq *q = rxq;
2053         struct pp2_ppio_tc_params *tc_params;
2054         int i, num, tc, inq;
2055         struct pp2_hif *hif;
2056         unsigned int core_id = rte_lcore_id();
2057
2058         if (core_id == LCORE_ID_ANY)
2059                 core_id = rte_get_main_lcore();
2060
2061         if (!q)
2062                 return;
2063
2064         hif = mrvl_get_hif(q->priv, core_id);
2065
2066         if (!hif)
2067                 return;
2068
2069         tc = q->priv->rxq_map[q->queue_id].tc;
2070         inq = q->priv->rxq_map[q->queue_id].inq;
2071         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
2072         num = tc_params->inqs_params[inq].size;
2073         for (i = 0; i < num; i++) {
2074                 struct pp2_buff_inf inf;
2075                 uint64_t addr;
2076
2077                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
2078                 addr = cookie_addr_high | inf.cookie;
2079                 rte_pktmbuf_free((struct rte_mbuf *)addr);
2080         }
2081
2082         rte_free(q);
2083 }
2084
2085 /**
2086  * DPDK callback to configure the transmit queue.
2087  *
2088  * @param dev
2089  *   Pointer to Ethernet device structure.
2090  * @param idx
2091  *   Transmit queue index.
2092  * @param desc
2093  *   Number of descriptors to configure in the queue.
2094  * @param socket
2095  *   NUMA socket on which memory must be allocated.
2096  * @param conf
2097  *   Tx queue configuration parameters.
2098  *
2099  * @return
2100  *   0 on success, negative error value otherwise.
2101  */
2102 static int
2103 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
2104                     unsigned int socket,
2105                     const struct rte_eth_txconf *conf)
2106 {
2107         struct mrvl_priv *priv = dev->data->dev_private;
2108         struct mrvl_txq *txq;
2109
2110         if (dev->data->tx_queues[idx]) {
2111                 rte_free(dev->data->tx_queues[idx]);
2112                 dev->data->tx_queues[idx] = NULL;
2113         }
2114
2115         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
2116         if (!txq)
2117                 return -ENOMEM;
2118
2119         txq->priv = priv;
2120         txq->queue_id = idx;
2121         txq->port_id = dev->data->port_id;
2122         txq->tx_deferred_start = conf->tx_deferred_start;
2123         dev->data->tx_queues[idx] = txq;
2124
2125         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
2126
2127         return 0;
2128 }
2129
2130 /**
2131  * DPDK callback to release the transmit queue.
2132  *
2133  * @param txq
2134  *   Generic transmit queue pointer.
2135  */
2136 static void
2137 mrvl_tx_queue_release(void *txq)
2138 {
2139         struct mrvl_txq *q = txq;
2140
2141         if (!q)
2142                 return;
2143
2144         rte_free(q);
2145 }
2146
2147 /**
2148  * DPDK callback to get flow control configuration.
2149  *
2150  * @param dev
2151  *  Pointer to Ethernet device structure.
2152  * @param fc_conf
2153  *  Pointer to the flow control configuration.
2154  *
2155  * @return
2156  *  0 on success, negative error value otherwise.
2157  */
2158 static int
2159 mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2160 {
2161         struct mrvl_priv *priv = dev->data->dev_private;
2162         int ret, en;
2163
2164         if (!priv->ppio) {
2165                 memcpy(fc_conf, &priv->fc_conf, sizeof(struct rte_eth_fc_conf));
2166                 return 0;
2167         }
2168
2169         fc_conf->autoneg = 1;
2170         ret = pp2_ppio_get_rx_pause(priv->ppio, &en);
2171         if (ret) {
2172                 MRVL_LOG(ERR, "Failed to read rx pause state");
2173                 return ret;
2174         }
2175
2176         fc_conf->mode = en ? RTE_FC_RX_PAUSE : RTE_FC_NONE;
2177
2178         ret = pp2_ppio_get_tx_pause(priv->ppio, &en);
2179         if (ret) {
2180                 MRVL_LOG(ERR, "Failed to read tx pause state");
2181                 return ret;
2182         }
2183
2184         if (en) {
2185                 if (fc_conf->mode == RTE_FC_NONE)
2186                         fc_conf->mode = RTE_FC_TX_PAUSE;
2187                 else
2188                         fc_conf->mode = RTE_FC_FULL;
2189         }
2190
2191         return 0;
2192 }
2193
2194 /**
2195  * DPDK callback to set flow control configuration.
2196  *
2197  * @param dev
2198  *  Pointer to Ethernet device structure.
2199  * @param fc_conf
2200  *  Pointer to the flow control configuration.
2201  *
2202  * @return
2203  *  0 on success, negative error value otherwise.
2204  */
2205 static int
2206 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2207 {
2208         struct mrvl_priv *priv = dev->data->dev_private;
2209         struct pp2_ppio_tx_pause_params mrvl_pause_params;
2210         int ret;
2211         int rx_en, tx_en;
2212
2213         if (fc_conf->high_water ||
2214             fc_conf->low_water ||
2215             fc_conf->pause_time ||
2216             fc_conf->mac_ctrl_frame_fwd) {
2217                 MRVL_LOG(ERR, "Flowctrl parameter is not supported");
2218
2219                 return -EINVAL;
2220         }
2221
2222         if (fc_conf->autoneg == 0) {
2223                 MRVL_LOG(ERR, "Flowctrl Autoneg disable is not supported");
2224                 return -EINVAL;
2225         }
2226
2227         if (!priv->ppio) {
2228                 memcpy(&priv->fc_conf, fc_conf, sizeof(struct rte_eth_fc_conf));
2229                 priv->flow_ctrl = 1;
2230                 return 0;
2231         }
2232
2233         switch (fc_conf->mode) {
2234         case RTE_FC_FULL:
2235                 rx_en = 1;
2236                 tx_en = 1;
2237                 break;
2238         case RTE_FC_TX_PAUSE:
2239                 rx_en = 0;
2240                 tx_en = 1;
2241                 break;
2242         case RTE_FC_RX_PAUSE:
2243                 rx_en = 1;
2244                 tx_en = 0;
2245                 break;
2246         case RTE_FC_NONE:
2247                 rx_en = 0;
2248                 tx_en = 0;
2249                 break;
2250         default:
2251                 MRVL_LOG(ERR, "Incorrect Flow control flag (%d)",
2252                          fc_conf->mode);
2253                 return -EINVAL;
2254         }
2255
2256         /* Set RX flow control */
2257         ret = pp2_ppio_set_rx_pause(priv->ppio, rx_en);
2258         if (ret) {
2259                 MRVL_LOG(ERR, "Failed to change RX flowctrl");
2260                 return ret;
2261         }
2262
2263         /* Set TX flow control */
2264         mrvl_pause_params.en = tx_en;
2265         /* all inqs participate in xon/xoff decision */
2266         mrvl_pause_params.use_tc_pause_inqs = 0;
2267         ret = pp2_ppio_set_tx_pause(priv->ppio, &mrvl_pause_params);
2268         if (ret) {
2269                 MRVL_LOG(ERR, "Failed to change TX flowctrl");
2270                 return ret;
2271         }
2272
2273         return 0;
2274 }
2275
2276 /**
2277  * Update RSS hash configuration
2278  *
2279  * @param dev
2280  *   Pointer to Ethernet device structure.
2281  * @param rss_conf
2282  *   Pointer to RSS configuration.
2283  *
2284  * @return
2285  *   0 on success, negative error value otherwise.
2286  */
2287 static int
2288 mrvl_rss_hash_update(struct rte_eth_dev *dev,
2289                      struct rte_eth_rss_conf *rss_conf)
2290 {
2291         struct mrvl_priv *priv = dev->data->dev_private;
2292
2293         if (priv->isolated)
2294                 return -ENOTSUP;
2295
2296         return mrvl_configure_rss(priv, rss_conf);
2297 }
2298
2299 /**
2300  * DPDK callback to get RSS hash configuration.
2301  *
2302  * @param dev
2303  *   Pointer to Ethernet device structure.
2304  * @rss_conf
2305  *   Pointer to RSS configuration.
2306  *
2307  * @return
2308  *   Always 0.
2309  */
2310 static int
2311 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
2312                        struct rte_eth_rss_conf *rss_conf)
2313 {
2314         struct mrvl_priv *priv = dev->data->dev_private;
2315         enum pp2_ppio_hash_type hash_type =
2316                 priv->ppio_params.inqs_params.hash_type;
2317
2318         rss_conf->rss_key = NULL;
2319
2320         if (hash_type == PP2_PPIO_HASH_T_NONE)
2321                 rss_conf->rss_hf = 0;
2322         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
2323                 rss_conf->rss_hf = ETH_RSS_IPV4;
2324         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
2325                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
2326         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
2327                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
2328
2329         return 0;
2330 }
2331
2332 /**
2333  * DPDK callback to get rte_flow callbacks.
2334  *
2335  * @param dev
2336  *   Pointer to the device structure.
2337  * @param filer_type
2338  *   Flow filter type.
2339  * @param filter_op
2340  *   Flow filter operation.
2341  * @param arg
2342  *   Pointer to pass the flow ops.
2343  *
2344  * @return
2345  *   0 on success, negative error value otherwise.
2346  */
2347 static int
2348 mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
2349                      enum rte_filter_type filter_type,
2350                      enum rte_filter_op filter_op, void *arg)
2351 {
2352         switch (filter_type) {
2353         case RTE_ETH_FILTER_GENERIC:
2354                 if (filter_op != RTE_ETH_FILTER_GET)
2355                         return -EINVAL;
2356                 *(const void **)arg = &mrvl_flow_ops;
2357                 return 0;
2358         default:
2359                 MRVL_LOG(WARNING, "Filter type (%d) not supported",
2360                                 filter_type);
2361                 return -EINVAL;
2362         }
2363 }
2364
2365 /**
2366  * DPDK callback to get rte_mtr callbacks.
2367  *
2368  * @param dev
2369  *   Pointer to the device structure.
2370  * @param ops
2371  *   Pointer to pass the mtr ops.
2372  *
2373  * @return
2374  *   Always 0.
2375  */
2376 static int
2377 mrvl_mtr_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
2378 {
2379         *(const void **)ops = &mrvl_mtr_ops;
2380
2381         return 0;
2382 }
2383
2384 /**
2385  * DPDK callback to get rte_tm callbacks.
2386  *
2387  * @param dev
2388  *   Pointer to the device structure.
2389  * @param ops
2390  *   Pointer to pass the tm ops.
2391  *
2392  * @return
2393  *   Always 0.
2394  */
2395 static int
2396 mrvl_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
2397 {
2398         *(const void **)ops = &mrvl_tm_ops;
2399
2400         return 0;
2401 }
2402
2403 static const struct eth_dev_ops mrvl_ops = {
2404         .dev_configure = mrvl_dev_configure,
2405         .dev_start = mrvl_dev_start,
2406         .dev_stop = mrvl_dev_stop,
2407         .dev_set_link_up = mrvl_dev_set_link_up,
2408         .dev_set_link_down = mrvl_dev_set_link_down,
2409         .dev_close = mrvl_dev_close,
2410         .link_update = mrvl_link_update,
2411         .promiscuous_enable = mrvl_promiscuous_enable,
2412         .allmulticast_enable = mrvl_allmulticast_enable,
2413         .promiscuous_disable = mrvl_promiscuous_disable,
2414         .allmulticast_disable = mrvl_allmulticast_disable,
2415         .mac_addr_remove = mrvl_mac_addr_remove,
2416         .mac_addr_add = mrvl_mac_addr_add,
2417         .mac_addr_set = mrvl_mac_addr_set,
2418         .mtu_set = mrvl_mtu_set,
2419         .stats_get = mrvl_stats_get,
2420         .stats_reset = mrvl_stats_reset,
2421         .xstats_get = mrvl_xstats_get,
2422         .xstats_reset = mrvl_xstats_reset,
2423         .xstats_get_names = mrvl_xstats_get_names,
2424         .dev_infos_get = mrvl_dev_infos_get,
2425         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
2426         .rxq_info_get = mrvl_rxq_info_get,
2427         .txq_info_get = mrvl_txq_info_get,
2428         .vlan_filter_set = mrvl_vlan_filter_set,
2429         .vlan_offload_set = mrvl_vlan_offload_set,
2430         .tx_queue_start = mrvl_tx_queue_start,
2431         .tx_queue_stop = mrvl_tx_queue_stop,
2432         .rx_queue_setup = mrvl_rx_queue_setup,
2433         .rx_queue_release = mrvl_rx_queue_release,
2434         .tx_queue_setup = mrvl_tx_queue_setup,
2435         .tx_queue_release = mrvl_tx_queue_release,
2436         .flow_ctrl_get = mrvl_flow_ctrl_get,
2437         .flow_ctrl_set = mrvl_flow_ctrl_set,
2438         .rss_hash_update = mrvl_rss_hash_update,
2439         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
2440         .filter_ctrl = mrvl_eth_filter_ctrl,
2441         .mtr_ops_get = mrvl_mtr_ops_get,
2442         .tm_ops_get = mrvl_tm_ops_get,
2443 };
2444
2445 /**
2446  * Return packet type information and l3/l4 offsets.
2447  *
2448  * @param desc
2449  *   Pointer to the received packet descriptor.
2450  * @param l3_offset
2451  *   l3 packet offset.
2452  * @param l4_offset
2453  *   l4 packet offset.
2454  *
2455  * @return
2456  *   Packet type information.
2457  */
2458 static inline uint64_t
2459 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
2460                                     uint8_t *l3_offset, uint8_t *l4_offset)
2461 {
2462         enum pp2_inq_l3_type l3_type;
2463         enum pp2_inq_l4_type l4_type;
2464         enum pp2_inq_vlan_tag vlan_tag;
2465         uint64_t packet_type;
2466
2467         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
2468         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
2469         pp2_ppio_inq_desc_get_vlan_tag(desc, &vlan_tag);
2470
2471         packet_type = RTE_PTYPE_L2_ETHER;
2472
2473         switch (vlan_tag) {
2474         case PP2_INQ_VLAN_TAG_SINGLE:
2475                 packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
2476                 break;
2477         case PP2_INQ_VLAN_TAG_DOUBLE:
2478         case PP2_INQ_VLAN_TAG_TRIPLE:
2479                 packet_type |= RTE_PTYPE_L2_ETHER_QINQ;
2480                 break;
2481         default:
2482                 break;
2483         }
2484
2485         switch (l3_type) {
2486         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
2487                 packet_type |= RTE_PTYPE_L3_IPV4;
2488                 break;
2489         case PP2_INQ_L3_TYPE_IPV4_OK:
2490                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
2491                 break;
2492         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
2493                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
2494                 break;
2495         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
2496                 packet_type |= RTE_PTYPE_L3_IPV6;
2497                 break;
2498         case PP2_INQ_L3_TYPE_IPV6_EXT:
2499                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
2500                 break;
2501         case PP2_INQ_L3_TYPE_ARP:
2502                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
2503                 /*
2504                  * In case of ARP l4_offset is set to wrong value.
2505                  * Set it to proper one so that later on mbuf->l3_len can be
2506                  * calculated subtracting l4_offset and l3_offset.
2507                  */
2508                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
2509                 break;
2510         default:
2511                 break;
2512         }
2513
2514         switch (l4_type) {
2515         case PP2_INQ_L4_TYPE_TCP:
2516                 packet_type |= RTE_PTYPE_L4_TCP;
2517                 break;
2518         case PP2_INQ_L4_TYPE_UDP:
2519                 packet_type |= RTE_PTYPE_L4_UDP;
2520                 break;
2521         default:
2522                 break;
2523         }
2524
2525         return packet_type;
2526 }
2527
2528 /**
2529  * Get offload information from the received packet descriptor.
2530  *
2531  * @param desc
2532  *   Pointer to the received packet descriptor.
2533  *
2534  * @return
2535  *   Mbuf offload flags.
2536  */
2537 static inline uint64_t
2538 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
2539 {
2540         uint64_t flags;
2541         enum pp2_inq_desc_status status;
2542
2543         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
2544         if (unlikely(status != PP2_DESC_ERR_OK))
2545                 flags = PKT_RX_IP_CKSUM_BAD;
2546         else
2547                 flags = PKT_RX_IP_CKSUM_GOOD;
2548
2549         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
2550         if (unlikely(status != PP2_DESC_ERR_OK))
2551                 flags |= PKT_RX_L4_CKSUM_BAD;
2552         else
2553                 flags |= PKT_RX_L4_CKSUM_GOOD;
2554
2555         return flags;
2556 }
2557
2558 /**
2559  * DPDK callback for receive.
2560  *
2561  * @param rxq
2562  *   Generic pointer to the receive queue.
2563  * @param rx_pkts
2564  *   Array to store received packets.
2565  * @param nb_pkts
2566  *   Maximum number of packets in array.
2567  *
2568  * @return
2569  *   Number of packets successfully received.
2570  */
2571 static uint16_t
2572 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2573 {
2574         struct mrvl_rxq *q = rxq;
2575         struct pp2_ppio_desc descs[nb_pkts];
2576         struct pp2_bpool *bpool;
2577         int i, ret, rx_done = 0;
2578         int num;
2579         struct pp2_hif *hif;
2580         unsigned int core_id = rte_lcore_id();
2581
2582         hif = mrvl_get_hif(q->priv, core_id);
2583
2584         if (unlikely(!q->priv->ppio || !hif))
2585                 return 0;
2586
2587         bpool = q->priv->bpool;
2588
2589         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
2590                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
2591         if (unlikely(ret < 0))
2592                 return 0;
2593
2594         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
2595
2596         for (i = 0; i < nb_pkts; i++) {
2597                 struct rte_mbuf *mbuf;
2598                 uint8_t l3_offset, l4_offset;
2599                 enum pp2_inq_desc_status status;
2600                 uint64_t addr;
2601
2602                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2603                         struct pp2_ppio_desc *pref_desc;
2604                         u64 pref_addr;
2605
2606                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
2607                         pref_addr = cookie_addr_high |
2608                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
2609                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
2610                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
2611                 }
2612
2613                 addr = cookie_addr_high |
2614                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
2615                 mbuf = (struct rte_mbuf *)addr;
2616                 rte_pktmbuf_reset(mbuf);
2617
2618                 /* drop packet in case of mac, overrun or resource error */
2619                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
2620                 if (unlikely(status != PP2_DESC_ERR_OK)) {
2621                         struct pp2_buff_inf binf = {
2622                                 .addr = rte_mbuf_data_iova_default(mbuf),
2623                                 .cookie = (uint64_t)mbuf,
2624                         };
2625
2626                         pp2_bpool_put_buff(hif, bpool, &binf);
2627                         mrvl_port_bpool_size
2628                                 [bpool->pp2_id][bpool->id][core_id]++;
2629                         q->drop_mac++;
2630                         continue;
2631                 }
2632
2633                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
2634                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
2635                 mbuf->data_len = mbuf->pkt_len;
2636                 mbuf->port = q->port_id;
2637                 mbuf->packet_type =
2638                         mrvl_desc_to_packet_type_and_offset(&descs[i],
2639                                                             &l3_offset,
2640                                                             &l4_offset);
2641                 mbuf->l2_len = l3_offset;
2642                 mbuf->l3_len = l4_offset - l3_offset;
2643
2644                 if (likely(q->cksum_enabled))
2645                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
2646
2647                 rx_pkts[rx_done++] = mbuf;
2648                 q->bytes_recv += mbuf->pkt_len;
2649         }
2650
2651         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
2652                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
2653
2654                 if (unlikely(num <= q->priv->bpool_min_size ||
2655                              (!rx_done && num < q->priv->bpool_init_size))) {
2656                         mrvl_fill_bpool(q, MRVL_BURST_SIZE);
2657                 } else if (unlikely(num > q->priv->bpool_max_size)) {
2658                         int i;
2659                         int pkt_to_remove = num - q->priv->bpool_init_size;
2660                         struct rte_mbuf *mbuf;
2661                         struct pp2_buff_inf buff;
2662
2663                         for (i = 0; i < pkt_to_remove; i++) {
2664                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
2665                                 if (ret)
2666                                         break;
2667                                 mbuf = (struct rte_mbuf *)
2668                                         (cookie_addr_high | buff.cookie);
2669                                 rte_pktmbuf_free(mbuf);
2670                         }
2671                         mrvl_port_bpool_size
2672                                 [bpool->pp2_id][bpool->id][core_id] -= i;
2673                 }
2674                 rte_spinlock_unlock(&q->priv->lock);
2675         }
2676
2677         return rx_done;
2678 }
2679
2680 /**
2681  * Prepare offload information.
2682  *
2683  * @param ol_flags
2684  *   Offload flags.
2685  * @param l3_type
2686  *   Pointer to the pp2_ouq_l3_type structure.
2687  * @param l4_type
2688  *   Pointer to the pp2_outq_l4_type structure.
2689  * @param gen_l3_cksum
2690  *   Will be set to 1 in case l3 checksum is computed.
2691  * @param l4_cksum
2692  *   Will be set to 1 in case l4 checksum is computed.
2693  */
2694 static inline void
2695 mrvl_prepare_proto_info(uint64_t ol_flags,
2696                         enum pp2_outq_l3_type *l3_type,
2697                         enum pp2_outq_l4_type *l4_type,
2698                         int *gen_l3_cksum,
2699                         int *gen_l4_cksum)
2700 {
2701         /*
2702          * Based on ol_flags prepare information
2703          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
2704          * for offloading.
2705          * in most of the checksum cases ipv4 must be set, so this is the
2706          * default value
2707          */
2708         *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
2709         *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
2710
2711         if (ol_flags & PKT_TX_IPV6) {
2712                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
2713                 /* no checksum for ipv6 header */
2714                 *gen_l3_cksum = 0;
2715         }
2716
2717         if ((ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) {
2718                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
2719                 *gen_l4_cksum = 1;
2720         } else if ((ol_flags & PKT_TX_L4_MASK) ==  PKT_TX_UDP_CKSUM) {
2721                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
2722                 *gen_l4_cksum = 1;
2723         } else {
2724                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
2725                 /* no checksum for other type */
2726                 *gen_l4_cksum = 0;
2727         }
2728 }
2729
2730 /**
2731  * Release already sent buffers to bpool (buffer-pool).
2732  *
2733  * @param ppio
2734  *   Pointer to the port structure.
2735  * @param hif
2736  *   Pointer to the MUSDK hardware interface.
2737  * @param sq
2738  *   Pointer to the shadow queue.
2739  * @param qid
2740  *   Queue id number.
2741  * @param force
2742  *   Force releasing packets.
2743  */
2744 static inline void
2745 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
2746                        unsigned int core_id, struct mrvl_shadow_txq *sq,
2747                        int qid, int force)
2748 {
2749         struct buff_release_entry *entry;
2750         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
2751         int i;
2752
2753         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
2754
2755         sq->num_to_release += nb_done;
2756
2757         if (likely(!force &&
2758                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
2759                 return;
2760
2761         nb_done = sq->num_to_release;
2762         sq->num_to_release = 0;
2763
2764         for (i = 0; i < nb_done; i++) {
2765                 entry = &sq->ent[sq->tail + num];
2766                 if (unlikely(!entry->buff.addr)) {
2767                         MRVL_LOG(ERR,
2768                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!",
2769                                 sq->tail, (u64)entry->buff.cookie,
2770                                 (u64)entry->buff.addr);
2771                         skip_bufs = 1;
2772                         goto skip;
2773                 }
2774
2775                 if (unlikely(!entry->bpool)) {
2776                         struct rte_mbuf *mbuf;
2777
2778                         mbuf = (struct rte_mbuf *)entry->buff.cookie;
2779                         rte_pktmbuf_free(mbuf);
2780                         skip_bufs = 1;
2781                         goto skip;
2782                 }
2783
2784                 mrvl_port_bpool_size
2785                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
2786                 num++;
2787                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
2788                         goto skip;
2789                 continue;
2790 skip:
2791                 if (likely(num))
2792                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2793                 num += skip_bufs;
2794                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2795                 sq->size -= num;
2796                 num = 0;
2797                 skip_bufs = 0;
2798         }
2799
2800         if (likely(num)) {
2801                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2802                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2803                 sq->size -= num;
2804         }
2805 }
2806
2807 /**
2808  * DPDK callback for transmit.
2809  *
2810  * @param txq
2811  *   Generic pointer transmit queue.
2812  * @param tx_pkts
2813  *   Packets to transmit.
2814  * @param nb_pkts
2815  *   Number of packets in array.
2816  *
2817  * @return
2818  *   Number of packets successfully transmitted.
2819  */
2820 static uint16_t
2821 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2822 {
2823         struct mrvl_txq *q = txq;
2824         struct mrvl_shadow_txq *sq;
2825         struct pp2_hif *hif;
2826         struct pp2_ppio_desc descs[nb_pkts];
2827         unsigned int core_id = rte_lcore_id();
2828         int i, bytes_sent = 0;
2829         uint16_t num, sq_free_size;
2830         uint64_t addr;
2831
2832         hif = mrvl_get_hif(q->priv, core_id);
2833         sq = &q->shadow_txqs[core_id];
2834
2835         if (unlikely(!q->priv->ppio || !hif))
2836                 return 0;
2837
2838         if (sq->size)
2839                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2840                                        sq, q->queue_id, 0);
2841
2842         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2843         if (unlikely(nb_pkts > sq_free_size))
2844                 nb_pkts = sq_free_size;
2845
2846         for (i = 0; i < nb_pkts; i++) {
2847                 struct rte_mbuf *mbuf = tx_pkts[i];
2848                 int gen_l3_cksum, gen_l4_cksum;
2849                 enum pp2_outq_l3_type l3_type;
2850                 enum pp2_outq_l4_type l4_type;
2851
2852                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2853                         struct rte_mbuf *pref_pkt_hdr;
2854
2855                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2856                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2857                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2858                 }
2859
2860                 mrvl_fill_shadowq(sq, mbuf);
2861                 mrvl_fill_desc(&descs[i], mbuf);
2862
2863                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2864                 /*
2865                  * in case unsupported ol_flags were passed
2866                  * do not update descriptor offload information
2867                  */
2868                 if (!(mbuf->ol_flags & MRVL_TX_PKT_OFFLOADS))
2869                         continue;
2870                 mrvl_prepare_proto_info(mbuf->ol_flags, &l3_type, &l4_type,
2871                                         &gen_l3_cksum, &gen_l4_cksum);
2872
2873                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2874                                                   mbuf->l2_len,
2875                                                   mbuf->l2_len + mbuf->l3_len,
2876                                                   gen_l3_cksum, gen_l4_cksum);
2877         }
2878
2879         num = nb_pkts;
2880         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2881         /* number of packets that were not sent */
2882         if (unlikely(num > nb_pkts)) {
2883                 for (i = nb_pkts; i < num; i++) {
2884                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2885                                 MRVL_PP2_TX_SHADOWQ_MASK;
2886                         addr = sq->ent[sq->head].buff.cookie;
2887                         bytes_sent -=
2888                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2889                 }
2890                 sq->size -= num - nb_pkts;
2891         }
2892
2893         q->bytes_sent += bytes_sent;
2894
2895         return nb_pkts;
2896 }
2897
2898 /** DPDK callback for S/G transmit.
2899  *
2900  * @param txq
2901  *   Generic pointer transmit queue.
2902  * @param tx_pkts
2903  *   Packets to transmit.
2904  * @param nb_pkts
2905  *   Number of packets in array.
2906  *
2907  * @return
2908  *   Number of packets successfully transmitted.
2909  */
2910 static uint16_t
2911 mrvl_tx_sg_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
2912                      uint16_t nb_pkts)
2913 {
2914         struct mrvl_txq *q = txq;
2915         struct mrvl_shadow_txq *sq;
2916         struct pp2_hif *hif;
2917         struct pp2_ppio_desc descs[nb_pkts * PP2_PPIO_DESC_NUM_FRAGS];
2918         struct pp2_ppio_sg_pkts pkts;
2919         uint8_t frags[nb_pkts];
2920         unsigned int core_id = rte_lcore_id();
2921         int i, j, bytes_sent = 0;
2922         int tail, tail_first;
2923         uint16_t num, sq_free_size;
2924         uint16_t nb_segs, total_descs = 0;
2925         uint64_t addr;
2926
2927         hif = mrvl_get_hif(q->priv, core_id);
2928         sq = &q->shadow_txqs[core_id];
2929         pkts.frags = frags;
2930         pkts.num = 0;
2931
2932         if (unlikely(!q->priv->ppio || !hif))
2933                 return 0;
2934
2935         if (sq->size)
2936                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2937                                        sq, q->queue_id, 0);
2938
2939         /* Save shadow queue free size */
2940         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2941
2942         tail = 0;
2943         for (i = 0; i < nb_pkts; i++) {
2944                 struct rte_mbuf *mbuf = tx_pkts[i];
2945                 struct rte_mbuf *seg = NULL;
2946                 int gen_l3_cksum, gen_l4_cksum;
2947                 enum pp2_outq_l3_type l3_type;
2948                 enum pp2_outq_l4_type l4_type;
2949
2950                 nb_segs = mbuf->nb_segs;
2951                 tail_first = tail;
2952                 total_descs += nb_segs;
2953
2954                 /*
2955                  * Check if total_descs does not exceed
2956                  * shadow queue free size
2957                  */
2958                 if (unlikely(total_descs > sq_free_size)) {
2959                         total_descs -= nb_segs;
2960                         break;
2961                 }
2962
2963                 /* Check if nb_segs does not exceed the max nb of desc per
2964                  * fragmented packet
2965                  */
2966                 if (nb_segs > PP2_PPIO_DESC_NUM_FRAGS) {
2967                         total_descs -= nb_segs;
2968                         RTE_LOG(ERR, PMD,
2969                                 "Too many segments. Packet won't be sent.\n");
2970                         break;
2971                 }
2972
2973                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2974                         struct rte_mbuf *pref_pkt_hdr;
2975
2976                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2977                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2978                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2979                 }
2980
2981                 pkts.frags[pkts.num] = nb_segs;
2982                 pkts.num++;
2983
2984                 seg = mbuf;
2985                 for (j = 0; j < nb_segs - 1; j++) {
2986                         /* For the subsequent segments, set shadow queue
2987                          * buffer to NULL
2988                          */
2989                         mrvl_fill_shadowq(sq, NULL);
2990                         mrvl_fill_desc(&descs[tail], seg);
2991
2992                         tail++;
2993                         seg = seg->next;
2994                 }
2995                 /* Put first mbuf info in last shadow queue entry */
2996                 mrvl_fill_shadowq(sq, mbuf);
2997                 /* Update descriptor with last segment */
2998                 mrvl_fill_desc(&descs[tail++], seg);
2999
3000                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
3001                 /* In case unsupported ol_flags were passed
3002                  * do not update descriptor offload information
3003                  */
3004                 if (!(mbuf->ol_flags & MRVL_TX_PKT_OFFLOADS))
3005                         continue;
3006                 mrvl_prepare_proto_info(mbuf->ol_flags, &l3_type, &l4_type,
3007                                         &gen_l3_cksum, &gen_l4_cksum);
3008
3009                 pp2_ppio_outq_desc_set_proto_info(&descs[tail_first], l3_type,
3010                                                   l4_type, mbuf->l2_len,
3011                                                   mbuf->l2_len + mbuf->l3_len,
3012                                                   gen_l3_cksum, gen_l4_cksum);
3013         }
3014
3015         num = total_descs;
3016         pp2_ppio_send_sg(q->priv->ppio, hif, q->queue_id, descs,
3017                          &total_descs, &pkts);
3018         /* number of packets that were not sent */
3019         if (unlikely(num > total_descs)) {
3020                 for (i = total_descs; i < num; i++) {
3021                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
3022                                 MRVL_PP2_TX_SHADOWQ_MASK;
3023
3024                         addr = sq->ent[sq->head].buff.cookie;
3025                         if (addr)
3026                                 bytes_sent -=
3027                                         rte_pktmbuf_pkt_len((struct rte_mbuf *)
3028                                                 (cookie_addr_high | addr));
3029                 }
3030                 sq->size -= num - total_descs;
3031                 nb_pkts = pkts.num;
3032         }
3033
3034         q->bytes_sent += bytes_sent;
3035
3036         return nb_pkts;
3037 }
3038
3039 /**
3040  * Create private device structure.
3041  *
3042  * @param dev_name
3043  *   Pointer to the port name passed in the initialization parameters.
3044  *
3045  * @return
3046  *   Pointer to the newly allocated private device structure.
3047  */
3048 static struct mrvl_priv *
3049 mrvl_priv_create(const char *dev_name)
3050 {
3051         struct pp2_bpool_params bpool_params;
3052         char match[MRVL_MATCH_LEN];
3053         struct mrvl_priv *priv;
3054         uint16_t max_frame_size;
3055         int ret, bpool_bit;
3056
3057         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
3058         if (!priv)
3059                 return NULL;
3060
3061         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
3062                                        &priv->pp_id, &priv->ppio_id);
3063         if (ret)
3064                 goto out_free_priv;
3065
3066         ret = pp2_ppio_get_l4_cksum_max_frame_size(priv->pp_id, priv->ppio_id,
3067                                                    &max_frame_size);
3068         if (ret)
3069                 goto out_free_priv;
3070
3071         priv->max_mtu = max_frame_size + RTE_ETHER_CRC_LEN -
3072                 MRVL_PP2_ETH_HDRS_LEN;
3073
3074         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
3075                                      PP2_BPOOL_NUM_POOLS);
3076         if (bpool_bit < 0)
3077                 goto out_free_priv;
3078         priv->bpool_bit = bpool_bit;
3079
3080         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
3081                  priv->bpool_bit);
3082         memset(&bpool_params, 0, sizeof(bpool_params));
3083         bpool_params.match = match;
3084         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
3085         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
3086         if (ret)
3087                 goto out_clear_bpool_bit;
3088
3089         priv->ppio_params.type = PP2_PPIO_T_NIC;
3090         rte_spinlock_init(&priv->lock);
3091
3092         return priv;
3093 out_clear_bpool_bit:
3094         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
3095 out_free_priv:
3096         rte_free(priv);
3097         return NULL;
3098 }
3099
3100 /**
3101  * Create device representing Ethernet port.
3102  *
3103  * @param name
3104  *   Pointer to the port's name.
3105  *
3106  * @return
3107  *   0 on success, negative error value otherwise.
3108  */
3109 static int
3110 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
3111 {
3112         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
3113         struct rte_eth_dev *eth_dev;
3114         struct mrvl_priv *priv;
3115         struct ifreq req;
3116
3117         eth_dev = rte_eth_dev_allocate(name);
3118         if (!eth_dev)
3119                 return -ENOMEM;
3120
3121         priv = mrvl_priv_create(name);
3122         if (!priv) {
3123                 ret = -ENOMEM;
3124                 goto out_free;
3125         }
3126         eth_dev->data->dev_private = priv;
3127
3128         eth_dev->data->mac_addrs =
3129                 rte_zmalloc("mac_addrs",
3130                             RTE_ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
3131         if (!eth_dev->data->mac_addrs) {
3132                 MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
3133                 ret = -ENOMEM;
3134                 goto out_free;
3135         }
3136
3137         memset(&req, 0, sizeof(req));
3138         strcpy(req.ifr_name, name);
3139         ret = ioctl(fd, SIOCGIFHWADDR, &req);
3140         if (ret)
3141                 goto out_free;
3142
3143         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
3144                req.ifr_addr.sa_data, RTE_ETHER_ADDR_LEN);
3145
3146         eth_dev->device = &vdev->device;
3147         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
3148         mrvl_set_tx_function(eth_dev);
3149         eth_dev->dev_ops = &mrvl_ops;
3150         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
3151
3152         eth_dev->data->dev_link.link_status = ETH_LINK_UP;
3153
3154         rte_eth_dev_probing_finish(eth_dev);
3155         return 0;
3156 out_free:
3157         rte_eth_dev_release_port(eth_dev);
3158
3159         return ret;
3160 }
3161
3162 /**
3163  * Callback used by rte_kvargs_process() during argument parsing.
3164  *
3165  * @param key
3166  *   Pointer to the parsed key (unused).
3167  * @param value
3168  *   Pointer to the parsed value.
3169  * @param extra_args
3170  *   Pointer to the extra arguments which contains address of the
3171  *   table of pointers to parsed interface names.
3172  *
3173  * @return
3174  *   Always 0.
3175  */
3176 static int
3177 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
3178                  void *extra_args)
3179 {
3180         struct mrvl_ifnames *ifnames = extra_args;
3181
3182         ifnames->names[ifnames->idx++] = value;
3183
3184         return 0;
3185 }
3186
3187 /**
3188  * DPDK callback to register the virtual device.
3189  *
3190  * @param vdev
3191  *   Pointer to the virtual device.
3192  *
3193  * @return
3194  *   0 on success, negative error value otherwise.
3195  */
3196 static int
3197 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
3198 {
3199         struct rte_kvargs *kvlist;
3200         struct mrvl_ifnames ifnames;
3201         int ret = -EINVAL;
3202         uint32_t i, ifnum, cfgnum;
3203         const char *params;
3204
3205         params = rte_vdev_device_args(vdev);
3206         if (!params)
3207                 return -EINVAL;
3208
3209         kvlist = rte_kvargs_parse(params, valid_args);
3210         if (!kvlist)
3211                 return -EINVAL;
3212
3213         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
3214         if (ifnum > RTE_DIM(ifnames.names))
3215                 goto out_free_kvlist;
3216
3217         ifnames.idx = 0;
3218         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
3219                            mrvl_get_ifnames, &ifnames);
3220
3221
3222         /*
3223          * The below system initialization should be done only once,
3224          * on the first provided configuration file
3225          */
3226         if (!mrvl_cfg) {
3227                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
3228                 MRVL_LOG(INFO, "Parsing config file!");
3229                 if (cfgnum > 1) {
3230                         MRVL_LOG(ERR, "Cannot handle more than one config file!");
3231                         goto out_free_kvlist;
3232                 } else if (cfgnum == 1) {
3233                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
3234                                            mrvl_get_cfg, &mrvl_cfg);
3235                 }
3236         }
3237
3238         if (mrvl_dev_num)
3239                 goto init_devices;
3240
3241         MRVL_LOG(INFO, "Perform MUSDK initializations");
3242
3243         ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist);
3244         if (ret)
3245                 goto out_free_kvlist;
3246
3247         ret = mrvl_init_pp2();
3248         if (ret) {
3249                 MRVL_LOG(ERR, "Failed to init PP!");
3250                 rte_mvep_deinit(MVEP_MOD_T_PP2);
3251                 goto out_free_kvlist;
3252         }
3253
3254         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
3255         memset(mrvl_port_to_bpool_lookup, 0, sizeof(mrvl_port_to_bpool_lookup));
3256
3257         mrvl_lcore_first = RTE_MAX_LCORE;
3258         mrvl_lcore_last = 0;
3259
3260 init_devices:
3261         for (i = 0; i < ifnum; i++) {
3262                 MRVL_LOG(INFO, "Creating %s", ifnames.names[i]);
3263                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
3264                 if (ret)
3265                         goto out_cleanup;
3266                 mrvl_dev_num++;
3267         }
3268
3269         rte_kvargs_free(kvlist);
3270
3271         return 0;
3272 out_cleanup:
3273         rte_pmd_mrvl_remove(vdev);
3274
3275 out_free_kvlist:
3276         rte_kvargs_free(kvlist);
3277
3278         return ret;
3279 }
3280
3281 /**
3282  * DPDK callback to remove virtual device.
3283  *
3284  * @param vdev
3285  *   Pointer to the removed virtual device.
3286  *
3287  * @return
3288  *   0 on success, negative error value otherwise.
3289  */
3290 static int
3291 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
3292 {
3293         uint16_t port_id;
3294         int ret = 0;
3295
3296         RTE_ETH_FOREACH_DEV(port_id) {
3297                 if (rte_eth_devices[port_id].device != &vdev->device)
3298                         continue;
3299                 ret |= rte_eth_dev_close(port_id);
3300         }
3301
3302         return ret == 0 ? 0 : -EIO;
3303 }
3304
3305 static struct rte_vdev_driver pmd_mrvl_drv = {
3306         .probe = rte_pmd_mrvl_probe,
3307         .remove = rte_pmd_mrvl_remove,
3308 };
3309
3310 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv);
3311 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2);
3312 RTE_LOG_REGISTER(mrvl_logtype, pmd.net.mvpp2, NOTICE);