drivers/net/mvpp2/mrvl_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017-2021 Marvell International Ltd.
   3  * Copyright(c) 2017-2021 Semihalf.
   4  * All rights reserved.
   5  */
   6
   7 #include <rte_string_fns.h>
   8 #include <ethdev_driver.h>
   9 #include <rte_kvargs.h>
  10 #include <rte_log.h>
  11 #include <rte_malloc.h>
  12 #include <rte_bus_vdev.h>
  13
  14 #include <fcntl.h>
  15 #include <linux/ethtool.h>
  16 #include <linux/sockios.h>
  17 #include <net/if.h>
  18 #include <net/if_arp.h>
  19 #include <sys/ioctl.h>
  20 #include <sys/socket.h>
  21 #include <sys/stat.h>
  22 #include <sys/types.h>
  23
  24 #include <rte_mvep_common.h>
  25 #include "mrvl_ethdev.h"
  26 #include "mrvl_qos.h"
  27 #include "mrvl_flow.h"
  28 #include "mrvl_mtr.h"
  29 #include "mrvl_tm.h"
  30
  31 /* bitmask with reserved hifs */
  32 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  33 /* bitmask with reserved bpools */
  34 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  35 /* bitmask with reserved kernel RSS tables */
  36 #define MRVL_MUSDK_RSS_RESERVED 0x0F
  37 /* maximum number of available hifs */
  38 #define MRVL_MUSDK_HIFS_MAX 9
  39
  40 /* prefetch shift */
  41 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  42
  43 /* TCAM has 25 entries reserved for uc/mc filter entries
  44  * + 1 for primary mac address
  45  */
  46 #define MRVL_MAC_ADDRS_MAX (1 + 25)
  47 #define MRVL_MATCH_LEN 16
  48 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  49 /* Maximum allowable packet size */
  50 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  51
  52 #define MRVL_IFACE_NAME_ARG "iface"
  53 #define MRVL_CFG_ARG "cfg"
  54
  55 #define MRVL_BURST_SIZE 64
  56
  57 #define MRVL_ARP_LENGTH 28
  58
  59 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  60 #define MRVL_COOKIE_HIGH_ADDR_MASK 0xffffff0000000000
  61
  62 /** Port Rx offload capabilities */
  63 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
  64                           DEV_RX_OFFLOAD_JUMBO_FRAME | \
  65                           DEV_RX_OFFLOAD_CHECKSUM)
  66
  67 /** Port Tx offloads capabilities */
  68 #define MRVL_TX_OFFLOAD_CHECKSUM (DEV_TX_OFFLOAD_IPV4_CKSUM | \
  69                                   DEV_TX_OFFLOAD_UDP_CKSUM  | \
  70                                   DEV_TX_OFFLOAD_TCP_CKSUM)
  71 #define MRVL_TX_OFFLOADS (MRVL_TX_OFFLOAD_CHECKSUM | \
  72                           DEV_TX_OFFLOAD_MULTI_SEGS)
  73
  74 #define MRVL_TX_PKT_OFFLOADS (PKT_TX_IP_CKSUM | \
  75                               PKT_TX_TCP_CKSUM | \
  76                               PKT_TX_UDP_CKSUM)
  77
  78 static const char * const valid_args[] = {
  79         MRVL_IFACE_NAME_ARG,
  80         MRVL_CFG_ARG,
  81         NULL
  82 };
  83
  84 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
  85 static struct pp2_hif *hifs[RTE_MAX_LCORE];
  86 static int used_bpools[PP2_NUM_PKT_PROC] = {
  87         [0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED
  88 };
  89
  90 static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
  91 static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
  92 static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
  93 static int dummy_pool_id[PP2_NUM_PKT_PROC];
  94 struct pp2_bpool *dummy_pool[PP2_NUM_PKT_PROC] = {0};
  95
  96 struct mrvl_ifnames {
  97         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
  98         int idx;
  99 };
 100
 101 /*
 102  * To use buffer harvesting based on loopback port shadow queue structure
 103  * was introduced for buffers information bookkeeping.
 104  *
 105  * Before sending the packet, related buffer information (pp2_buff_inf) is
 106  * stored in shadow queue. After packet is transmitted no longer used
 107  * packet buffer is released back to it's original hardware pool,
 108  * on condition it originated from interface.
 109  * In case it  was generated by application itself i.e: mbuf->port field is
 110  * 0xff then its released to software mempool.
 111  */
 112 struct mrvl_shadow_txq {
 113         int head;           /* write index - used when sending buffers */
 114         int tail;           /* read index - used when releasing buffers */
 115         u16 size;           /* queue occupied size */
 116         u16 num_to_release; /* number of descriptors sent, that can be
 117                              * released
 118                              */
 119         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 120 };
 121
 122 struct mrvl_rxq {
 123         struct mrvl_priv *priv;
 124         struct rte_mempool *mp;
 125         int queue_id;
 126         int port_id;
 127         int cksum_enabled;
 128         uint64_t bytes_recv;
 129         uint64_t drop_mac;
 130 };
 131
 132 struct mrvl_txq {
 133         struct mrvl_priv *priv;
 134         int queue_id;
 135         int port_id;
 136         uint64_t bytes_sent;
 137         struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE];
 138         int tx_deferred_start;
 139 };
 140
 141 static int mrvl_lcore_first;
 142 static int mrvl_lcore_last;
 143 static int mrvl_dev_num;
 144
 145 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
 146 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
 147                         struct pp2_hif *hif, unsigned int core_id,
 148                         struct mrvl_shadow_txq *sq, int qid, int force);
 149
 150 static uint16_t mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
 151                                   uint16_t nb_pkts);
 152 static uint16_t mrvl_tx_sg_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
 153                                      uint16_t nb_pkts);
 154 static int rte_pmd_mrvl_remove(struct rte_vdev_device *vdev);
 155 static void mrvl_deinit_pp2(void);
 156 static void mrvl_deinit_hifs(void);
 157
 158 static int
 159 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
 160                   uint32_t index, uint32_t vmdq __rte_unused);
 161 static int
 162 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr);
 163 static int
 164 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
 165 static int mrvl_promiscuous_enable(struct rte_eth_dev *dev);
 166 static int mrvl_allmulticast_enable(struct rte_eth_dev *dev);
 167 static int
 168 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf);
 169
 170 #define MRVL_XSTATS_TBL_ENTRY(name) { \
 171         #name, offsetof(struct pp2_ppio_statistics, name),      \
 172         sizeof(((struct pp2_ppio_statistics *)0)->name)         \
 173 }
 174
 175 /* Table with xstats data */
 176 static struct {
 177         const char *name;
 178         unsigned int offset;
 179         unsigned int size;
 180 } mrvl_xstats_tbl[] = {
 181         MRVL_XSTATS_TBL_ENTRY(rx_bytes),
 182         MRVL_XSTATS_TBL_ENTRY(rx_packets),
 183         MRVL_XSTATS_TBL_ENTRY(rx_unicast_packets),
 184         MRVL_XSTATS_TBL_ENTRY(rx_errors),
 185         MRVL_XSTATS_TBL_ENTRY(rx_fullq_dropped),
 186         MRVL_XSTATS_TBL_ENTRY(rx_bm_dropped),
 187         MRVL_XSTATS_TBL_ENTRY(rx_early_dropped),
 188         MRVL_XSTATS_TBL_ENTRY(rx_fifo_dropped),
 189         MRVL_XSTATS_TBL_ENTRY(rx_cls_dropped),
 190         MRVL_XSTATS_TBL_ENTRY(tx_bytes),
 191         MRVL_XSTATS_TBL_ENTRY(tx_packets),
 192         MRVL_XSTATS_TBL_ENTRY(tx_unicast_packets),
 193         MRVL_XSTATS_TBL_ENTRY(tx_errors)
 194 };
 195
 196 static inline int
 197 mrvl_reserve_bit(int *bitmap, int max)
 198 {
 199         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 200
 201         if (n >= max)
 202                 return -1;
 203
 204         *bitmap |= 1 << n;
 205
 206         return n;
 207 }
 208
 209 static int
 210 mrvl_pp2_fixup_init(void)
 211 {
 212         struct pp2_bpool_params bpool_params;
 213         char                    name[15];
 214         int                     err, i;
 215
 216         memset(dummy_pool, 0, sizeof(dummy_pool));
 217         for (i = 0; i < pp2_get_num_inst(); i++) {
 218                 dummy_pool_id[i] = mrvl_reserve_bit(&used_bpools[i],
 219                                              PP2_BPOOL_NUM_POOLS);
 220                 if (dummy_pool_id[i] < 0) {
 221                         MRVL_LOG(ERR, "Can't find free pool\n");
 222                         return -1;
 223                 }
 224
 225                 memset(name, 0, sizeof(name));
 226                 snprintf(name, sizeof(name), "pool-%d:%d", i, dummy_pool_id[i]);
 227                 memset(&bpool_params, 0, sizeof(bpool_params));
 228                 bpool_params.match = name;
 229                 bpool_params.buff_len = MRVL_PKT_OFFS;
 230                 bpool_params.dummy_short_pool = 1;
 231                 err = pp2_bpool_init(&bpool_params, &dummy_pool[i]);
 232                 if (err != 0 || !dummy_pool[i]) {
 233                         MRVL_LOG(ERR, "BPool init failed!\n");
 234                         used_bpools[i] &= ~(1 << dummy_pool_id[i]);
 235                         return -1;
 236                 }
 237         }
 238
 239         return 0;
 240 }
 241
 242 /**
 243  * Initialize packet processor.
 244  *
 245  * @return
 246  *   0 on success, negative error value otherwise.
 247  */
 248 static int
 249 mrvl_init_pp2(void)
 250 {
 251         struct pp2_init_params  init_params;
 252         int                     err;
 253
 254         memset(&init_params, 0, sizeof(init_params));
 255         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
 256         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
 257         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
 258         if (mrvl_cfg && mrvl_cfg->pp2_cfg.prs_udfs.num_udfs)
 259                 memcpy(&init_params.prs_udfs, &mrvl_cfg->pp2_cfg.prs_udfs,
 260                        sizeof(struct pp2_parse_udfs));
 261         err = pp2_init(&init_params);
 262         if (err != 0) {
 263                 MRVL_LOG(ERR, "PP2 init failed");
 264                 return -1;
 265         }
 266
 267         err = mrvl_pp2_fixup_init();
 268         if (err != 0) {
 269                 MRVL_LOG(ERR, "PP2 fixup init failed");
 270                 return -1;
 271         }
 272
 273         return 0;
 274 }
 275
 276 static void
 277 mrvl_pp2_fixup_deinit(void)
 278 {
 279         int i;
 280
 281         for (i = 0; i < PP2_NUM_PKT_PROC; i++) {
 282                 if (!dummy_pool[i])
 283                         continue;
 284                 pp2_bpool_deinit(dummy_pool[i]);
 285                 used_bpools[i] &= ~(1 << dummy_pool_id[i]);
 286         }
 287 }
 288
 289 /**
 290  * Deinitialize packet processor.
 291  *
 292  * @return
 293  *   0 on success, negative error value otherwise.
 294  */
 295 static void
 296 mrvl_deinit_pp2(void)
 297 {
 298         mrvl_pp2_fixup_deinit();
 299         pp2_deinit();
 300 }
 301
 302 static inline void
 303 mrvl_fill_shadowq(struct mrvl_shadow_txq *sq, struct rte_mbuf *buf)
 304 {
 305         sq->ent[sq->head].buff.cookie = (uint64_t)buf;
 306         sq->ent[sq->head].buff.addr = buf ?
 307                 rte_mbuf_data_iova_default(buf) : 0;
 308
 309         sq->ent[sq->head].bpool =
 310                 (unlikely(!buf || buf->port >= RTE_MAX_ETHPORTS ||
 311                  buf->refcnt > 1)) ? NULL :
 312                  mrvl_port_to_bpool_lookup[buf->port];
 313
 314         sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
 315         sq->size++;
 316 }
 317
 318 /**
 319  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
 320  */
 321 static void
 322 mrvl_deinit_hifs(void)
 323 {
 324         int i;
 325
 326         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
 327                 if (hifs[i])
 328                         pp2_hif_deinit(hifs[i]);
 329         }
 330         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
 331         memset(hifs, 0, sizeof(hifs));
 332 }
 333
 334 static inline void
 335 mrvl_fill_desc(struct pp2_ppio_desc *desc, struct rte_mbuf *buf)
 336 {
 337         pp2_ppio_outq_desc_reset(desc);
 338         pp2_ppio_outq_desc_set_phys_addr(desc, rte_pktmbuf_iova(buf));
 339         pp2_ppio_outq_desc_set_pkt_offset(desc, 0);
 340         pp2_ppio_outq_desc_set_pkt_len(desc, rte_pktmbuf_data_len(buf));
 341 }
 342
 343 static inline int
 344 mrvl_get_bpool_size(int pp2_id, int pool_id)
 345 {
 346         int i;
 347         int size = 0;
 348
 349         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 350                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 351
 352         return size;
 353 }
 354
 355 static int
 356 mrvl_init_hif(int core_id)
 357 {
 358         struct pp2_hif_params params;
 359         char match[MRVL_MATCH_LEN];
 360         int ret;
 361
 362         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
 363         if (ret < 0) {
 364                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 365                 return ret;
 366         }
 367
 368         snprintf(match, sizeof(match), "hif-%d", ret);
 369         memset(&params, 0, sizeof(params));
 370         params.match = match;
 371         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
 372         ret = pp2_hif_init(&params, &hifs[core_id]);
 373         if (ret) {
 374                 MRVL_LOG(ERR, "Failed to initialize hif %d", core_id);
 375                 return ret;
 376         }
 377
 378         return 0;
 379 }
 380
 381 static inline struct pp2_hif*
 382 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
 383 {
 384         int ret;
 385
 386         if (likely(hifs[core_id] != NULL))
 387                 return hifs[core_id];
 388
 389         rte_spinlock_lock(&priv->lock);
 390
 391         ret = mrvl_init_hif(core_id);
 392         if (ret < 0) {
 393                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 394                 goto out;
 395         }
 396
 397         if (core_id < mrvl_lcore_first)
 398                 mrvl_lcore_first = core_id;
 399
 400         if (core_id > mrvl_lcore_last)
 401                 mrvl_lcore_last = core_id;
 402 out:
 403         rte_spinlock_unlock(&priv->lock);
 404
 405         return hifs[core_id];
 406 }
 407
 408 /**
 409  * Set tx burst function according to offload flag
 410  *
 411  * @param dev
 412  *   Pointer to Ethernet device structure.
 413  */
 414 static void
 415 mrvl_set_tx_function(struct rte_eth_dev *dev)
 416 {
 417         struct mrvl_priv *priv = dev->data->dev_private;
 418
 419         /* Use a simple Tx queue (no offloads, no multi segs) if possible */
 420         if (priv->multiseg) {
 421                 RTE_LOG(INFO, PMD, "Using multi-segment tx callback\n");
 422                 dev->tx_pkt_burst = mrvl_tx_sg_pkt_burst;
 423         } else {
 424                 RTE_LOG(INFO, PMD, "Using single-segment tx callback\n");
 425                 dev->tx_pkt_burst = mrvl_tx_pkt_burst;
 426         }
 427 }
 428
 429 /**
 430  * Configure rss based on dpdk rss configuration.
 431  *
 432  * @param priv
 433  *   Pointer to private structure.
 434  * @param rss_conf
 435  *   Pointer to RSS configuration.
 436  *
 437  * @return
 438  *   0 on success, negative error value otherwise.
 439  */
 440 static int
 441 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 442 {
 443         if (rss_conf->rss_key)
 444                 MRVL_LOG(WARNING, "Changing hash key is not supported");
 445
 446         if (rss_conf->rss_hf == 0) {
 447                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 448         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 449                 priv->ppio_params.inqs_params.hash_type =
 450                         PP2_PPIO_HASH_T_2_TUPLE;
 451         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 452                 priv->ppio_params.inqs_params.hash_type =
 453                         PP2_PPIO_HASH_T_5_TUPLE;
 454                 priv->rss_hf_tcp = 1;
 455         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 456                 priv->ppio_params.inqs_params.hash_type =
 457                         PP2_PPIO_HASH_T_5_TUPLE;
 458                 priv->rss_hf_tcp = 0;
 459         } else {
 460                 return -EINVAL;
 461         }
 462
 463         return 0;
 464 }
 465
 466 /**
 467  * Ethernet device configuration.
 468  *
 469  * Prepare the driver for a given number of TX and RX queues and
 470  * configure RSS.
 471  *
 472  * @param dev
 473  *   Pointer to Ethernet device structure.
 474  *
 475  * @return
 476  *   0 on success, negative error value otherwise.
 477  */
 478 static int
 479 mrvl_dev_configure(struct rte_eth_dev *dev)
 480 {
 481         struct mrvl_priv *priv = dev->data->dev_private;
 482         int ret;
 483
 484         if (priv->ppio) {
 485                 MRVL_LOG(INFO, "Device reconfiguration is not supported");
 486                 return -EINVAL;
 487         }
 488
 489         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 490             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 491                 MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
 492                         dev->data->dev_conf.rxmode.mq_mode);
 493                 return -EINVAL;
 494         }
 495
 496         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 497                 MRVL_LOG(INFO, "Split headers not supported");
 498                 return -EINVAL;
 499         }
 500
 501         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
 502                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 503                                  MRVL_PP2_ETH_HDRS_LEN;
 504                 if (dev->data->mtu > priv->max_mtu) {
 505                         MRVL_LOG(ERR, "inherit MTU %u from max_rx_pkt_len %u is larger than max_mtu %u\n",
 506                                  dev->data->mtu,
 507                                  dev->data->dev_conf.rxmode.max_rx_pkt_len,
 508                                  priv->max_mtu);
 509                         return -EINVAL;
 510                 }
 511         }
 512
 513         if (dev->data->dev_conf.txmode.offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
 514                 priv->multiseg = 1;
 515
 516         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 517                                   dev->data->nb_rx_queues);
 518         if (ret < 0)
 519                 return ret;
 520
 521         ret = mrvl_configure_txqs(priv, dev->data->port_id,
 522                                   dev->data->nb_tx_queues);
 523         if (ret < 0)
 524                 return ret;
 525
 526         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 527         priv->ppio_params.maintain_stats = 1;
 528         priv->nb_rx_queues = dev->data->nb_rx_queues;
 529
 530         ret = mrvl_tm_init(dev);
 531         if (ret < 0)
 532                 return ret;
 533
 534         if (dev->data->nb_rx_queues == 1 &&
 535             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 536                 MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
 537                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 538
 539                 return 0;
 540         }
 541
 542         return mrvl_configure_rss(priv,
 543                         &dev->data->dev_conf.rx_adv_conf.rss_conf);
 544 }
 545
 546 /**
 547  * DPDK callback to change the MTU.
 548  *
 549  * Setting the MTU affects hardware MRU (packets larger than the MRU
 550  * will be dropped).
 551  *
 552  * @param dev
 553  *   Pointer to Ethernet device structure.
 554  * @param mtu
 555  *   New MTU.
 556  *
 557  * @return
 558  *   0 on success, negative error value otherwise.
 559  */
 560 static int
 561 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 562 {
 563         struct mrvl_priv *priv = dev->data->dev_private;
 564         uint16_t mru;
 565         uint16_t mbuf_data_size = 0; /* SW buffer size */
 566         int ret;
 567
 568         mru = MRVL_PP2_MTU_TO_MRU(mtu);
 569         /*
 570          * min_rx_buf_size is equal to mbuf data size
 571          * if pmd didn't set it differently
 572          */
 573         mbuf_data_size = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
 574         /* Prevent PMD from:
 575          * - setting mru greater than the mbuf size resulting in
 576          * hw and sw buffer size mismatch
 577          * - setting mtu that requires the support of scattered packets
 578          * when this feature has not been enabled/supported so far
 579          * (TODO check scattered_rx flag here once scattered RX is supported).
 580          */
 581         if (mru - RTE_ETHER_CRC_LEN + MRVL_PKT_OFFS > mbuf_data_size) {
 582                 mru = mbuf_data_size + RTE_ETHER_CRC_LEN - MRVL_PKT_OFFS;
 583                 mtu = MRVL_PP2_MRU_TO_MTU(mru);
 584                 MRVL_LOG(WARNING, "MTU too big, max MTU possible limitted "
 585                         "by current mbuf size: %u. Set MTU to %u, MRU to %u",
 586                         mbuf_data_size, mtu, mru);
 587         }
 588
 589         if (mtu < RTE_ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX) {
 590                 MRVL_LOG(ERR, "Invalid MTU [%u] or MRU [%u]", mtu, mru);
 591                 return -EINVAL;
 592         }
 593
 594         dev->data->mtu = mtu;
 595         dev->data->dev_conf.rxmode.max_rx_pkt_len = mru - MV_MH_SIZE;
 596
 597         if (!priv->ppio)
 598                 return 0;
 599
 600         ret = pp2_ppio_set_mru(priv->ppio, mru);
 601         if (ret) {
 602                 MRVL_LOG(ERR, "Failed to change MRU");
 603                 return ret;
 604         }
 605
 606         ret = pp2_ppio_set_mtu(priv->ppio, mtu);
 607         if (ret) {
 608                 MRVL_LOG(ERR, "Failed to change MTU");
 609                 return ret;
 610         }
 611
 612         return 0;
 613 }
 614
 615 /**
 616  * DPDK callback to bring the link up.
 617  *
 618  * @param dev
 619  *   Pointer to Ethernet device structure.
 620  *
 621  * @return
 622  *   0 on success, negative error value otherwise.
 623  */
 624 static int
 625 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 626 {
 627         struct mrvl_priv *priv = dev->data->dev_private;
 628         int ret;
 629
 630         if (!priv->ppio) {
 631                 dev->data->dev_link.link_status = ETH_LINK_UP;
 632                 return 0;
 633         }
 634
 635         ret = pp2_ppio_enable(priv->ppio);
 636         if (ret)
 637                 return ret;
 638
 639         /*
 640          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 641          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 642          * PP2_TRAFFIC_INGRESS_EGRESS.
 643          *
 644          * Set mtu to default DPDK value here.
 645          */
 646         ret = mrvl_mtu_set(dev, dev->data->mtu);
 647         if (ret) {
 648                 pp2_ppio_disable(priv->ppio);
 649                 return ret;
 650         }
 651
 652         dev->data->dev_link.link_status = ETH_LINK_UP;
 653         return 0;
 654 }
 655
 656 /**
 657  * DPDK callback to bring the link down.
 658  *
 659  * @param dev
 660  *   Pointer to Ethernet device structure.
 661  *
 662  * @return
 663  *   0 on success, negative error value otherwise.
 664  */
 665 static int
 666 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 667 {
 668         struct mrvl_priv *priv = dev->data->dev_private;
 669         int ret;
 670
 671         if (!priv->ppio) {
 672                 dev->data->dev_link.link_status = ETH_LINK_DOWN;
 673                 return 0;
 674         }
 675         ret = pp2_ppio_disable(priv->ppio);
 676         if (ret)
 677                 return ret;
 678
 679         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 680         return 0;
 681 }
 682
 683 /**
 684  * DPDK callback to start tx queue.
 685  *
 686  * @param dev
 687  *   Pointer to Ethernet device structure.
 688  * @param queue_id
 689  *   Transmit queue index.
 690  *
 691  * @return
 692  *   0 on success, negative error value otherwise.
 693  */
 694 static int
 695 mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
 696 {
 697         struct mrvl_priv *priv = dev->data->dev_private;
 698         int ret;
 699
 700         if (!priv)
 701                 return -EPERM;
 702
 703         /* passing 1 enables given tx queue */
 704         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1);
 705         if (ret) {
 706                 MRVL_LOG(ERR, "Failed to start txq %d", queue_id);
 707                 return ret;
 708         }
 709
 710         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 711
 712         return 0;
 713 }
 714
 715 /**
 716  * DPDK callback to stop tx queue.
 717  *
 718  * @param dev
 719  *   Pointer to Ethernet device structure.
 720  * @param queue_id
 721  *   Transmit queue index.
 722  *
 723  * @return
 724  *   0 on success, negative error value otherwise.
 725  */
 726 static int
 727 mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
 728 {
 729         struct mrvl_priv *priv = dev->data->dev_private;
 730         int ret;
 731
 732         if (!priv->ppio)
 733                 return -EPERM;
 734
 735         /* passing 0 disables given tx queue */
 736         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0);
 737         if (ret) {
 738                 MRVL_LOG(ERR, "Failed to stop txq %d", queue_id);
 739                 return ret;
 740         }
 741
 742         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 743
 744         return 0;
 745 }
 746
 747 /**
 748  * Populate VLAN Filter configuration.
 749  *
 750  * @param dev
 751  *   Pointer to Ethernet device structure.
 752  * @param on
 753  *   Toggle filter.
 754  *
 755  * @return
 756  *   0 on success, negative error value otherwise.
 757  */
 758 static int mrvl_populate_vlan_table(struct rte_eth_dev *dev, int on)
 759 {
 760         uint32_t j;
 761         int ret;
 762         struct rte_vlan_filter_conf *vfc;
 763
 764         vfc = &dev->data->vlan_filter_conf;
 765         for (j = 0; j < RTE_DIM(vfc->ids); j++) {
 766                 uint64_t vlan;
 767                 uint64_t vbit;
 768                 uint64_t ids = vfc->ids[j];
 769
 770                 if (ids == 0)
 771                         continue;
 772
 773                 while (ids) {
 774                         vlan = 64 * j;
 775                         /* count trailing zeroes */
 776                         vbit = ~ids & (ids - 1);
 777                         /* clear least significant bit set */
 778                         ids ^= (ids ^ (ids - 1)) ^ vbit;
 779                         for (; vbit; vlan++)
 780                                 vbit >>= 1;
 781                         ret = mrvl_vlan_filter_set(dev, vlan, on);
 782                         if (ret) {
 783                                 MRVL_LOG(ERR, "Failed to setup VLAN filter\n");
 784                                 return ret;
 785                         }
 786                 }
 787         }
 788
 789         return 0;
 790 }
 791
 792 /**
 793  * DPDK callback to start the device.
 794  *
 795  * @param dev
 796  *   Pointer to Ethernet device structure.
 797  *
 798  * @return
 799  *   0 on success, negative errno value on failure.
 800  */
 801 static int
 802 mrvl_dev_start(struct rte_eth_dev *dev)
 803 {
 804         struct mrvl_priv *priv = dev->data->dev_private;
 805         char match[MRVL_MATCH_LEN];
 806         int ret = 0, i, def_init_size;
 807         struct rte_ether_addr *mac_addr;
 808
 809         if (priv->ppio)
 810                 return mrvl_dev_set_link_up(dev);
 811
 812         snprintf(match, sizeof(match), "ppio-%d:%d",
 813                  priv->pp_id, priv->ppio_id);
 814         priv->ppio_params.match = match;
 815         priv->ppio_params.eth_start_hdr = PP2_PPIO_HDR_ETH;
 816         priv->forward_bad_frames = 0;
 817
 818         if (mrvl_cfg) {
 819                 priv->ppio_params.eth_start_hdr =
 820                         mrvl_cfg->port[dev->data->port_id].eth_start_hdr;
 821                 priv->forward_bad_frames =
 822                         mrvl_cfg->port[dev->data->port_id].forward_bad_frames;
 823         }
 824
 825         /*
 826          * Calculate the minimum bpool size for refill feature as follows:
 827          * 2 default burst sizes multiply by number of rx queues.
 828          * If the bpool size will be below this value, new buffers will
 829          * be added to the pool.
 830          */
 831         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 832
 833         /* In case initial bpool size configured in queues setup is
 834          * smaller than minimum size add more buffers
 835          */
 836         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
 837         if (priv->bpool_init_size < def_init_size) {
 838                 int buffs_to_add = def_init_size - priv->bpool_init_size;
 839
 840                 priv->bpool_init_size += buffs_to_add;
 841                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
 842                 if (ret)
 843                         MRVL_LOG(ERR, "Failed to add buffers to bpool");
 844         }
 845
 846         /*
 847          * Calculate the maximum bpool size for refill feature as follows:
 848          * maximum number of descriptors in rx queue multiply by number
 849          * of rx queues plus minimum bpool size.
 850          * In case the bpool size will exceed this value, superfluous buffers
 851          * will be removed
 852          */
 853         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
 854                                 priv->bpool_min_size;
 855
 856         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 857         if (ret) {
 858                 MRVL_LOG(ERR, "Failed to init ppio");
 859                 return ret;
 860         }
 861
 862         /*
 863          * In case there are some some stale uc/mc mac addresses flush them
 864          * here. It cannot be done during mrvl_dev_close() as port information
 865          * is already gone at that point (due to pp2_ppio_deinit() in
 866          * mrvl_dev_stop()).
 867          */
 868         if (!priv->uc_mc_flushed) {
 869                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 870                 if (ret) {
 871                         MRVL_LOG(ERR,
 872                                 "Failed to flush uc/mc filter list");
 873                         goto out;
 874                 }
 875                 priv->uc_mc_flushed = 1;
 876         }
 877
 878         ret = mrvl_mtu_set(dev, dev->data->mtu);
 879         if (ret)
 880                 MRVL_LOG(ERR, "Failed to set MTU to %d", dev->data->mtu);
 881
 882         if (!rte_is_zero_ether_addr(&dev->data->mac_addrs[0]))
 883                 mrvl_mac_addr_set(dev, &dev->data->mac_addrs[0]);
 884
 885         for (i = 1; i < MRVL_MAC_ADDRS_MAX; i++) {
 886                 mac_addr = &dev->data->mac_addrs[i];
 887
 888                 /* skip zero address */
 889                 if (rte_is_zero_ether_addr(mac_addr))
 890                         continue;
 891
 892                 mrvl_mac_addr_add(dev, mac_addr, i, 0);
 893         }
 894
 895         if (dev->data->all_multicast == 1)
 896                 mrvl_allmulticast_enable(dev);
 897
 898         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_VLAN_FILTER) {
 899                 ret = mrvl_populate_vlan_table(dev, 1);
 900                 if (ret) {
 901                         MRVL_LOG(ERR, "Failed to populate VLAN table");
 902                         goto out;
 903                 }
 904         }
 905
 906         /* For default QoS config, don't start classifier. */
 907         if (mrvl_cfg  &&
 908             mrvl_cfg->port[dev->data->port_id].use_qos_global_defaults == 0) {
 909                 ret = mrvl_start_qos_mapping(priv);
 910                 if (ret) {
 911                         MRVL_LOG(ERR, "Failed to setup QoS mapping");
 912                         goto out;
 913                 }
 914         }
 915
 916         ret = pp2_ppio_set_loopback(priv->ppio, dev->data->dev_conf.lpbk_mode);
 917         if (ret) {
 918                 MRVL_LOG(ERR, "Failed to set loopback");
 919                 goto out;
 920         }
 921
 922         if (dev->data->promiscuous == 1)
 923                 mrvl_promiscuous_enable(dev);
 924
 925         if (priv->flow_ctrl) {
 926                 ret = mrvl_flow_ctrl_set(dev, &priv->fc_conf);
 927                 if (ret) {
 928                         MRVL_LOG(ERR, "Failed to configure flow control");
 929                         goto out;
 930                 }
 931                 priv->flow_ctrl = 0;
 932         }
 933
 934         if (dev->data->dev_link.link_status == ETH_LINK_UP) {
 935                 ret = mrvl_dev_set_link_up(dev);
 936                 if (ret) {
 937                         MRVL_LOG(ERR, "Failed to set link up");
 938                         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 939                         goto out;
 940                 }
 941         }
 942
 943         /* start tx queues */
 944         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 945                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 946
 947                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
 948
 949                 if (!txq->tx_deferred_start)
 950                         continue;
 951
 952                 /*
 953                  * All txqs are started by default. Stop them
 954                  * so that tx_deferred_start works as expected.
 955                  */
 956                 ret = mrvl_tx_queue_stop(dev, i);
 957                 if (ret)
 958                         goto out;
 959         }
 960
 961         mrvl_flow_init(dev);
 962         mrvl_mtr_init(dev);
 963         mrvl_set_tx_function(dev);
 964
 965         return 0;
 966 out:
 967         MRVL_LOG(ERR, "Failed to start device");
 968         pp2_ppio_deinit(priv->ppio);
 969         return ret;
 970 }
 971
 972 /**
 973  * Flush receive queues.
 974  *
 975  * @param dev
 976  *   Pointer to Ethernet device structure.
 977  */
 978 static void
 979 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 980 {
 981         int i;
 982
 983         MRVL_LOG(INFO, "Flushing rx queues");
 984         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 985                 int ret, num;
 986
 987                 do {
 988                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 989                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 990
 991                         num = MRVL_PP2_RXD_MAX;
 992                         ret = pp2_ppio_recv(q->priv->ppio,
 993                                             q->priv->rxq_map[q->queue_id].tc,
 994                                             q->priv->rxq_map[q->queue_id].inq,
 995                                             descs, (uint16_t *)&num);
 996                 } while (ret == 0 && num);
 997         }
 998 }
 999
1000 /**
1001  * Flush transmit shadow queues.
1002  *
1003  * @param dev
1004  *   Pointer to Ethernet device structure.
1005  */
1006 static void
1007 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
1008 {
1009         int i, j;
1010         struct mrvl_txq *txq;
1011
1012         MRVL_LOG(INFO, "Flushing tx shadow queues");
1013         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1014                 txq = (struct mrvl_txq *)dev->data->tx_queues[i];
1015
1016                 for (j = 0; j < RTE_MAX_LCORE; j++) {
1017                         struct mrvl_shadow_txq *sq;
1018
1019                         if (!hifs[j])
1020                                 continue;
1021
1022                         sq = &txq->shadow_txqs[j];
1023                         mrvl_free_sent_buffers(txq->priv->ppio,
1024                                 hifs[j], j, sq, txq->queue_id, 1);
1025                         while (sq->tail != sq->head) {
1026                                 uint64_t addr = cookie_addr_high |
1027                                         sq->ent[sq->tail].buff.cookie;
1028                                 rte_pktmbuf_free(
1029                                         (struct rte_mbuf *)addr);
1030                                 sq->tail = (sq->tail + 1) &
1031                                             MRVL_PP2_TX_SHADOWQ_MASK;
1032                         }
1033                         memset(sq, 0, sizeof(*sq));
1034                 }
1035         }
1036 }
1037
1038 /**
1039  * Flush hardware bpool (buffer-pool).
1040  *
1041  * @param dev
1042  *   Pointer to Ethernet device structure.
1043  */
1044 static void
1045 mrvl_flush_bpool(struct rte_eth_dev *dev)
1046 {
1047         struct mrvl_priv *priv = dev->data->dev_private;
1048         struct pp2_hif *hif;
1049         uint32_t num;
1050         int ret;
1051         unsigned int core_id = rte_lcore_id();
1052
1053         if (core_id == LCORE_ID_ANY)
1054                 core_id = rte_get_main_lcore();
1055
1056         hif = mrvl_get_hif(priv, core_id);
1057
1058         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
1059         if (ret) {
1060                 MRVL_LOG(ERR, "Failed to get bpool buffers number");
1061                 return;
1062         }
1063
1064         while (num--) {
1065                 struct pp2_buff_inf inf;
1066                 uint64_t addr;
1067
1068                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
1069                 if (ret)
1070                         break;
1071
1072                 addr = cookie_addr_high | inf.cookie;
1073                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1074         }
1075 }
1076
1077 /**
1078  * DPDK callback to stop the device.
1079  *
1080  * @param dev
1081  *   Pointer to Ethernet device structure.
1082  */
1083 static int
1084 mrvl_dev_stop(struct rte_eth_dev *dev)
1085 {
1086         return mrvl_dev_set_link_down(dev);
1087 }
1088
1089 /**
1090  * DPDK callback to close the device.
1091  *
1092  * @param dev
1093  *   Pointer to Ethernet device structure.
1094  */
1095 static int
1096 mrvl_dev_close(struct rte_eth_dev *dev)
1097 {
1098         struct mrvl_priv *priv = dev->data->dev_private;
1099         size_t i;
1100
1101         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1102                 return 0;
1103
1104         mrvl_flush_rx_queues(dev);
1105         mrvl_flush_tx_shadow_queues(dev);
1106         mrvl_flow_deinit(dev);
1107         mrvl_mtr_deinit(dev);
1108
1109         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
1110                 struct pp2_ppio_tc_params *tc_params =
1111                         &priv->ppio_params.inqs_params.tcs_params[i];
1112
1113                 if (tc_params->inqs_params) {
1114                         rte_free(tc_params->inqs_params);
1115                         tc_params->inqs_params = NULL;
1116                 }
1117         }
1118
1119         if (priv->cls_tbl) {
1120                 pp2_cls_tbl_deinit(priv->cls_tbl);
1121                 priv->cls_tbl = NULL;
1122         }
1123
1124         if (priv->qos_tbl) {
1125                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
1126                 priv->qos_tbl = NULL;
1127         }
1128
1129         mrvl_flush_bpool(dev);
1130         mrvl_tm_deinit(dev);
1131
1132         if (priv->ppio) {
1133                 pp2_ppio_deinit(priv->ppio);
1134                 priv->ppio = NULL;
1135         }
1136
1137         /* policer must be released after ppio deinitialization */
1138         if (priv->default_policer) {
1139                 pp2_cls_plcr_deinit(priv->default_policer);
1140                 priv->default_policer = NULL;
1141         }
1142
1143
1144         if (priv->bpool) {
1145                 pp2_bpool_deinit(priv->bpool);
1146                 used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
1147                 priv->bpool = NULL;
1148         }
1149
1150         mrvl_dev_num--;
1151
1152         if (mrvl_dev_num == 0) {
1153                 MRVL_LOG(INFO, "Perform MUSDK deinit");
1154                 mrvl_deinit_hifs();
1155                 mrvl_deinit_pp2();
1156                 rte_mvep_deinit(MVEP_MOD_T_PP2);
1157         }
1158
1159         return 0;
1160 }
1161
1162 /**
1163  * DPDK callback to retrieve physical link information.
1164  *
1165  * @param dev
1166  *   Pointer to Ethernet device structure.
1167  * @param wait_to_complete
1168  *   Wait for request completion (ignored).
1169  *
1170  * @return
1171  *   0 on success, negative error value otherwise.
1172  */
1173 static int
1174 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
1175 {
1176         /*
1177          * TODO
1178          * once MUSDK provides necessary API use it here
1179          */
1180         struct mrvl_priv *priv = dev->data->dev_private;
1181         struct ethtool_cmd edata;
1182         struct ifreq req;
1183         int ret, fd, link_up;
1184
1185         if (!priv->ppio)
1186                 return -EPERM;
1187
1188         edata.cmd = ETHTOOL_GSET;
1189
1190         strcpy(req.ifr_name, dev->data->name);
1191         req.ifr_data = (void *)&edata;
1192
1193         fd = socket(AF_INET, SOCK_DGRAM, 0);
1194         if (fd == -1)
1195                 return -EFAULT;
1196
1197         ret = ioctl(fd, SIOCETHTOOL, &req);
1198         if (ret == -1) {
1199                 close(fd);
1200                 return -EFAULT;
1201         }
1202
1203         close(fd);
1204
1205         switch (ethtool_cmd_speed(&edata)) {
1206         case SPEED_10:
1207                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
1208                 break;
1209         case SPEED_100:
1210                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
1211                 break;
1212         case SPEED_1000:
1213                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
1214                 break;
1215         case SPEED_2500:
1216                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_2_5G;
1217                 break;
1218         case SPEED_10000:
1219                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
1220                 break;
1221         default:
1222                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
1223         }
1224
1225         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
1226                                                          ETH_LINK_HALF_DUPLEX;
1227         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
1228                                                            ETH_LINK_FIXED;
1229         pp2_ppio_get_link_state(priv->ppio, &link_up);
1230         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
1231
1232         return 0;
1233 }
1234
1235 /**
1236  * DPDK callback to enable promiscuous mode.
1237  *
1238  * @param dev
1239  *   Pointer to Ethernet device structure.
1240  *
1241  * @return
1242  *   0 on success, negative error value otherwise.
1243  */
1244 static int
1245 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
1246 {
1247         struct mrvl_priv *priv = dev->data->dev_private;
1248         int ret;
1249
1250         if (priv->isolated)
1251                 return -ENOTSUP;
1252
1253         if (!priv->ppio)
1254                 return 0;
1255
1256         ret = pp2_ppio_set_promisc(priv->ppio, 1);
1257         if (ret) {
1258                 MRVL_LOG(ERR, "Failed to enable promiscuous mode");
1259                 return -EAGAIN;
1260         }
1261
1262         return 0;
1263 }
1264
1265 /**
1266  * DPDK callback to enable allmulti mode.
1267  *
1268  * @param dev
1269  *   Pointer to Ethernet device structure.
1270  *
1271  * @return
1272  *   0 on success, negative error value otherwise.
1273  */
1274 static int
1275 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
1276 {
1277         struct mrvl_priv *priv = dev->data->dev_private;
1278         int ret;
1279
1280         if (priv->isolated)
1281                 return -ENOTSUP;
1282
1283         if (!priv->ppio)
1284                 return 0;
1285
1286         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
1287         if (ret) {
1288                 MRVL_LOG(ERR, "Failed enable all-multicast mode");
1289                 return -EAGAIN;
1290         }
1291
1292         return 0;
1293 }
1294
1295 /**
1296  * DPDK callback to disable promiscuous mode.
1297  *
1298  * @param dev
1299  *   Pointer to Ethernet device structure.
1300  *
1301  * @return
1302  *   0 on success, negative error value otherwise.
1303  */
1304 static int
1305 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
1306 {
1307         struct mrvl_priv *priv = dev->data->dev_private;
1308         int ret;
1309
1310         if (priv->isolated)
1311                 return -ENOTSUP;
1312
1313         if (!priv->ppio)
1314                 return 0;
1315
1316         ret = pp2_ppio_set_promisc(priv->ppio, 0);
1317         if (ret) {
1318                 MRVL_LOG(ERR, "Failed to disable promiscuous mode");
1319                 return -EAGAIN;
1320         }
1321
1322         return 0;
1323 }
1324
1325 /**
1326  * DPDK callback to disable allmulticast mode.
1327  *
1328  * @param dev
1329  *   Pointer to Ethernet device structure.
1330  *
1331  * @return
1332  *   0 on success, negative error value otherwise.
1333  */
1334 static int
1335 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
1336 {
1337         struct mrvl_priv *priv = dev->data->dev_private;
1338         int ret;
1339
1340         if (priv->isolated)
1341                 return -ENOTSUP;
1342
1343         if (!priv->ppio)
1344                 return 0;
1345
1346         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
1347         if (ret) {
1348                 MRVL_LOG(ERR, "Failed to disable all-multicast mode");
1349                 return -EAGAIN;
1350         }
1351
1352         return 0;
1353 }
1354
1355 /**
1356  * DPDK callback to remove a MAC address.
1357  *
1358  * @param dev
1359  *   Pointer to Ethernet device structure.
1360  * @param index
1361  *   MAC address index.
1362  */
1363 static void
1364 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1365 {
1366         struct mrvl_priv *priv = dev->data->dev_private;
1367         char buf[RTE_ETHER_ADDR_FMT_SIZE];
1368         int ret;
1369
1370         if (priv->isolated)
1371                 return;
1372
1373         if (!priv->ppio)
1374                 return;
1375
1376         ret = pp2_ppio_remove_mac_addr(priv->ppio,
1377                                        dev->data->mac_addrs[index].addr_bytes);
1378         if (ret) {
1379                 rte_ether_format_addr(buf, sizeof(buf),
1380                                   &dev->data->mac_addrs[index]);
1381                 MRVL_LOG(ERR, "Failed to remove mac %s", buf);
1382         }
1383 }
1384
1385 /**
1386  * DPDK callback to add a MAC address.
1387  *
1388  * @param dev
1389  *   Pointer to Ethernet device structure.
1390  * @param mac_addr
1391  *   MAC address to register.
1392  * @param index
1393  *   MAC address index.
1394  * @param vmdq
1395  *   VMDq pool index to associate address with (unused).
1396  *
1397  * @return
1398  *   0 on success, negative error value otherwise.
1399  */
1400 static int
1401 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1402                   uint32_t index, uint32_t vmdq __rte_unused)
1403 {
1404         struct mrvl_priv *priv = dev->data->dev_private;
1405         char buf[RTE_ETHER_ADDR_FMT_SIZE];
1406         int ret;
1407
1408         if (priv->isolated)
1409                 return -ENOTSUP;
1410
1411         if (!priv->ppio)
1412                 return 0;
1413
1414         if (index == 0)
1415                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
1416                 return -1;
1417
1418         /*
1419          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
1420          * parameter uc_filter_max. Maximum number of mc addresses is then
1421          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
1422          * 21 respectively.
1423          *
1424          * If more than uc_filter_max uc addresses were added to filter list
1425          * then NIC will switch to promiscuous mode automatically.
1426          *
1427          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
1428          * were added to filter list then NIC will switch to all-multicast mode
1429          * automatically.
1430          */
1431         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
1432         if (ret) {
1433                 rte_ether_format_addr(buf, sizeof(buf), mac_addr);
1434                 MRVL_LOG(ERR, "Failed to add mac %s", buf);
1435                 return -1;
1436         }
1437
1438         return 0;
1439 }
1440
1441 /**
1442  * DPDK callback to set the primary MAC address.
1443  *
1444  * @param dev
1445  *   Pointer to Ethernet device structure.
1446  * @param mac_addr
1447  *   MAC address to register.
1448  *
1449  * @return
1450  *   0 on success, negative error value otherwise.
1451  */
1452 static int
1453 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
1454 {
1455         struct mrvl_priv *priv = dev->data->dev_private;
1456         int ret;
1457
1458         if (priv->isolated)
1459                 return -ENOTSUP;
1460
1461         if (!priv->ppio)
1462                 return 0;
1463
1464         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
1465         if (ret) {
1466                 char buf[RTE_ETHER_ADDR_FMT_SIZE];
1467                 rte_ether_format_addr(buf, sizeof(buf), mac_addr);
1468                 MRVL_LOG(ERR, "Failed to set mac to %s", buf);
1469         }
1470
1471         return ret;
1472 }
1473
1474 /**
1475  * DPDK callback to get device statistics.
1476  *
1477  * @param dev
1478  *   Pointer to Ethernet device structure.
1479  * @param stats
1480  *   Stats structure output buffer.
1481  *
1482  * @return
1483  *   0 on success, negative error value otherwise.
1484  */
1485 static int
1486 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1487 {
1488         struct mrvl_priv *priv = dev->data->dev_private;
1489         struct pp2_ppio_statistics ppio_stats;
1490         uint64_t drop_mac = 0;
1491         unsigned int i, idx, ret;
1492
1493         if (!priv->ppio)
1494                 return -EPERM;
1495
1496         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1497                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1498                 struct pp2_ppio_inq_statistics rx_stats;
1499
1500                 if (!rxq)
1501                         continue;
1502
1503                 idx = rxq->queue_id;
1504                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1505                         MRVL_LOG(ERR,
1506                                 "rx queue %d stats out of range (0 - %d)",
1507                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1508                         continue;
1509                 }
1510
1511                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
1512                                                   priv->rxq_map[idx].tc,
1513                                                   priv->rxq_map[idx].inq,
1514                                                   &rx_stats, 0);
1515                 if (unlikely(ret)) {
1516                         MRVL_LOG(ERR,
1517                                 "Failed to update rx queue %d stats", idx);
1518                         break;
1519                 }
1520
1521                 stats->q_ibytes[idx] = rxq->bytes_recv;
1522                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
1523                 stats->q_errors[idx] = rx_stats.drop_early +
1524                                        rx_stats.drop_fullq +
1525                                        rx_stats.drop_bm +
1526                                        rxq->drop_mac;
1527                 stats->ibytes += rxq->bytes_recv;
1528                 drop_mac += rxq->drop_mac;
1529         }
1530
1531         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1532                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1533                 struct pp2_ppio_outq_statistics tx_stats;
1534
1535                 if (!txq)
1536                         continue;
1537
1538                 idx = txq->queue_id;
1539                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1540                         MRVL_LOG(ERR,
1541                                 "tx queue %d stats out of range (0 - %d)",
1542                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1543                 }
1544
1545                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1546                                                    &tx_stats, 0);
1547                 if (unlikely(ret)) {
1548                         MRVL_LOG(ERR,
1549                                 "Failed to update tx queue %d stats", idx);
1550                         break;
1551                 }
1552
1553                 stats->q_opackets[idx] = tx_stats.deq_desc;
1554                 stats->q_obytes[idx] = txq->bytes_sent;
1555                 stats->obytes += txq->bytes_sent;
1556         }
1557
1558         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1559         if (unlikely(ret)) {
1560                 MRVL_LOG(ERR, "Failed to update port statistics");
1561                 return ret;
1562         }
1563
1564         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1565         stats->opackets += ppio_stats.tx_packets;
1566         stats->imissed += ppio_stats.rx_fullq_dropped +
1567                           ppio_stats.rx_bm_dropped +
1568                           ppio_stats.rx_early_dropped +
1569                           ppio_stats.rx_fifo_dropped +
1570                           ppio_stats.rx_cls_dropped;
1571         stats->ierrors = drop_mac;
1572
1573         return 0;
1574 }
1575
1576 /**
1577  * DPDK callback to clear device statistics.
1578  *
1579  * @param dev
1580  *   Pointer to Ethernet device structure.
1581  *
1582  * @return
1583  *   0 on success, negative error value otherwise.
1584  */
1585 static int
1586 mrvl_stats_reset(struct rte_eth_dev *dev)
1587 {
1588         struct mrvl_priv *priv = dev->data->dev_private;
1589         int i;
1590
1591         if (!priv->ppio)
1592                 return 0;
1593
1594         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1595                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1596
1597                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1598                                             priv->rxq_map[i].inq, NULL, 1);
1599                 rxq->bytes_recv = 0;
1600                 rxq->drop_mac = 0;
1601         }
1602
1603         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1604                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1605
1606                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1607                 txq->bytes_sent = 0;
1608         }
1609
1610         return pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1611 }
1612
1613 /**
1614  * DPDK callback to get extended statistics.
1615  *
1616  * @param dev
1617  *   Pointer to Ethernet device structure.
1618  * @param stats
1619  *   Pointer to xstats table.
1620  * @param n
1621  *   Number of entries in xstats table.
1622  * @return
1623  *   Negative value on error, number of read xstats otherwise.
1624  */
1625 static int
1626 mrvl_xstats_get(struct rte_eth_dev *dev,
1627                 struct rte_eth_xstat *stats, unsigned int n)
1628 {
1629         struct mrvl_priv *priv = dev->data->dev_private;
1630         struct pp2_ppio_statistics ppio_stats;
1631         unsigned int i;
1632
1633         if (!stats)
1634                 return 0;
1635
1636         pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1637         for (i = 0; i < n && i < RTE_DIM(mrvl_xstats_tbl); i++) {
1638                 uint64_t val;
1639
1640                 if (mrvl_xstats_tbl[i].size == sizeof(uint32_t))
1641                         val = *(uint32_t *)((uint8_t *)&ppio_stats +
1642                                             mrvl_xstats_tbl[i].offset);
1643                 else if (mrvl_xstats_tbl[i].size == sizeof(uint64_t))
1644                         val = *(uint64_t *)((uint8_t *)&ppio_stats +
1645                                             mrvl_xstats_tbl[i].offset);
1646                 else
1647                         return -EINVAL;
1648
1649                 stats[i].id = i;
1650                 stats[i].value = val;
1651         }
1652
1653         return n;
1654 }
1655
1656 /**
1657  * DPDK callback to reset extended statistics.
1658  *
1659  * @param dev
1660  *   Pointer to Ethernet device structure.
1661  *
1662  * @return
1663  *   0 on success, negative error value otherwise.
1664  */
1665 static int
1666 mrvl_xstats_reset(struct rte_eth_dev *dev)
1667 {
1668         return mrvl_stats_reset(dev);
1669 }
1670
1671 /**
1672  * DPDK callback to get extended statistics names.
1673  *
1674  * @param dev (unused)
1675  *   Pointer to Ethernet device structure.
1676  * @param xstats_names
1677  *   Pointer to xstats names table.
1678  * @param size
1679  *   Size of the xstats names table.
1680  * @return
1681  *   Number of read names.
1682  */
1683 static int
1684 mrvl_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
1685                       struct rte_eth_xstat_name *xstats_names,
1686                       unsigned int size)
1687 {
1688         unsigned int i;
1689
1690         if (!xstats_names)
1691                 return RTE_DIM(mrvl_xstats_tbl);
1692
1693         for (i = 0; i < size && i < RTE_DIM(mrvl_xstats_tbl); i++)
1694                 strlcpy(xstats_names[i].name, mrvl_xstats_tbl[i].name,
1695                         RTE_ETH_XSTATS_NAME_SIZE);
1696
1697         return size;
1698 }
1699
1700 /**
1701  * DPDK callback to get information about the device.
1702  *
1703  * @param dev
1704  *   Pointer to Ethernet device structure (unused).
1705  * @param info
1706  *   Info structure output buffer.
1707  */
1708 static int
1709 mrvl_dev_infos_get(struct rte_eth_dev *dev,
1710                    struct rte_eth_dev_info *info)
1711 {
1712         struct mrvl_priv *priv = dev->data->dev_private;
1713
1714         info->speed_capa = ETH_LINK_SPEED_10M |
1715                            ETH_LINK_SPEED_100M |
1716                            ETH_LINK_SPEED_1G |
1717                            ETH_LINK_SPEED_2_5G |
1718                            ETH_LINK_SPEED_10G;
1719
1720         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1721         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1722         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1723
1724         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1725         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1726         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1727
1728         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1729         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1730         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1731
1732         info->rx_offload_capa = MRVL_RX_OFFLOADS;
1733         info->rx_queue_offload_capa = MRVL_RX_OFFLOADS;
1734
1735         info->tx_offload_capa = MRVL_TX_OFFLOADS;
1736         info->tx_queue_offload_capa = MRVL_TX_OFFLOADS;
1737
1738         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1739                                        ETH_RSS_NONFRAG_IPV4_TCP |
1740                                        ETH_RSS_NONFRAG_IPV4_UDP;
1741
1742         /* By default packets are dropped if no descriptors are available */
1743         info->default_rxconf.rx_drop_en = 1;
1744
1745         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1746         info->max_mtu = priv->max_mtu;
1747
1748         return 0;
1749 }
1750
1751 /**
1752  * Return supported packet types.
1753  *
1754  * @param dev
1755  *   Pointer to Ethernet device structure (unused).
1756  *
1757  * @return
1758  *   Const pointer to the table with supported packet types.
1759  */
1760 static const uint32_t *
1761 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1762 {
1763         static const uint32_t ptypes[] = {
1764                 RTE_PTYPE_L2_ETHER,
1765                 RTE_PTYPE_L2_ETHER_VLAN,
1766                 RTE_PTYPE_L2_ETHER_QINQ,
1767                 RTE_PTYPE_L3_IPV4,
1768                 RTE_PTYPE_L3_IPV4_EXT,
1769                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1770                 RTE_PTYPE_L3_IPV6,
1771                 RTE_PTYPE_L3_IPV6_EXT,
1772                 RTE_PTYPE_L2_ETHER_ARP,
1773                 RTE_PTYPE_L4_TCP,
1774                 RTE_PTYPE_L4_UDP
1775         };
1776
1777         return ptypes;
1778 }
1779
1780 /**
1781  * DPDK callback to get information about specific receive queue.
1782  *
1783  * @param dev
1784  *   Pointer to Ethernet device structure.
1785  * @param rx_queue_id
1786  *   Receive queue index.
1787  * @param qinfo
1788  *   Receive queue information structure.
1789  */
1790 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1791                               struct rte_eth_rxq_info *qinfo)
1792 {
1793         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1794         struct mrvl_priv *priv = dev->data->dev_private;
1795         int inq = priv->rxq_map[rx_queue_id].inq;
1796         int tc = priv->rxq_map[rx_queue_id].tc;
1797         struct pp2_ppio_tc_params *tc_params =
1798                 &priv->ppio_params.inqs_params.tcs_params[tc];
1799
1800         qinfo->mp = q->mp;
1801         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1802 }
1803
1804 /**
1805  * DPDK callback to get information about specific transmit queue.
1806  *
1807  * @param dev
1808  *   Pointer to Ethernet device structure.
1809  * @param tx_queue_id
1810  *   Transmit queue index.
1811  * @param qinfo
1812  *   Transmit queue information structure.
1813  */
1814 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1815                               struct rte_eth_txq_info *qinfo)
1816 {
1817         struct mrvl_priv *priv = dev->data->dev_private;
1818         struct mrvl_txq *txq = dev->data->tx_queues[tx_queue_id];
1819
1820         qinfo->nb_desc =
1821                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1822         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1823 }
1824
1825 /**
1826  * DPDK callback to Configure a VLAN filter.
1827  *
1828  * @param dev
1829  *   Pointer to Ethernet device structure.
1830  * @param vlan_id
1831  *   VLAN ID to filter.
1832  * @param on
1833  *   Toggle filter.
1834  *
1835  * @return
1836  *   0 on success, negative error value otherwise.
1837  */
1838 static int
1839 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1840 {
1841         struct mrvl_priv *priv = dev->data->dev_private;
1842
1843         if (priv->isolated)
1844                 return -ENOTSUP;
1845
1846         if (!priv->ppio)
1847                 return 0;
1848
1849         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1850                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1851 }
1852
1853 /**
1854  * DPDK callback to Configure VLAN offload.
1855  *
1856  * @param dev
1857  *   Pointer to Ethernet device structure.
1858  * @param mask
1859  *   VLAN offload mask.
1860  *
1861  * @return
1862  *   0 on success, negative error value otherwise.
1863  */
1864 static int mrvl_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1865 {
1866         uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
1867         int ret;
1868
1869         if (mask & ETH_VLAN_STRIP_MASK)
1870                 MRVL_LOG(ERR, "VLAN stripping is not supported\n");
1871
1872         if (mask & ETH_VLAN_FILTER_MASK) {
1873                 if (rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
1874                         ret = mrvl_populate_vlan_table(dev, 1);
1875                 else
1876                         ret = mrvl_populate_vlan_table(dev, 0);
1877
1878                 if (ret)
1879                         return ret;
1880         }
1881
1882         if (mask & ETH_VLAN_EXTEND_MASK)
1883                 MRVL_LOG(ERR, "Extend VLAN not supported\n");
1884
1885         return 0;
1886 }
1887
1888 /**
1889  * Release buffers to hardware bpool (buffer-pool)
1890  *
1891  * @param rxq
1892  *   Receive queue pointer.
1893  * @param num
1894  *   Number of buffers to release to bpool.
1895  *
1896  * @return
1897  *   0 on success, negative error value otherwise.
1898  */
1899 static int
1900 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1901 {
1902         struct buff_release_entry entries[num];
1903         struct rte_mbuf *mbufs[num];
1904         int i, ret;
1905         unsigned int core_id;
1906         struct pp2_hif *hif;
1907         struct pp2_bpool *bpool;
1908
1909         core_id = rte_lcore_id();
1910         if (core_id == LCORE_ID_ANY)
1911                 core_id = rte_get_main_lcore();
1912
1913         hif = mrvl_get_hif(rxq->priv, core_id);
1914         if (!hif)
1915                 return -1;
1916
1917         bpool = rxq->priv->bpool;
1918
1919         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1920         if (ret)
1921                 return ret;
1922
1923         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1924                 cookie_addr_high =
1925                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1926
1927         for (i = 0; i < num; i++) {
1928                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1929                         != cookie_addr_high) {
1930                         MRVL_LOG(ERR,
1931                                 "mbuf virtual addr high is out of range "
1932                                 "0x%x instead of 0x%x\n",
1933                                 (uint32_t)((uint64_t)mbufs[i] >> 32),
1934                                 (uint32_t)(cookie_addr_high >> 32));
1935                         goto out;
1936                 }
1937
1938                 entries[i].buff.addr =
1939                         rte_mbuf_data_iova_default(mbufs[i]);
1940                 entries[i].buff.cookie = (uintptr_t)mbufs[i];
1941                 entries[i].bpool = bpool;
1942         }
1943
1944         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1945         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1946
1947         if (i != num)
1948                 goto out;
1949
1950         return 0;
1951 out:
1952         for (; i < num; i++)
1953                 rte_pktmbuf_free(mbufs[i]);
1954
1955         return -1;
1956 }
1957
1958 /**
1959  * DPDK callback to configure the receive queue.
1960  *
1961  * @param dev
1962  *   Pointer to Ethernet device structure.
1963  * @param idx
1964  *   RX queue index.
1965  * @param desc
1966  *   Number of descriptors to configure in queue.
1967  * @param socket
1968  *   NUMA socket on which memory must be allocated.
1969  * @param conf
1970  *   Thresholds parameters.
1971  * @param mp
1972  *   Memory pool for buffer allocations.
1973  *
1974  * @return
1975  *   0 on success, negative error value otherwise.
1976  */
1977 static int
1978 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1979                     unsigned int socket,
1980                     const struct rte_eth_rxconf *conf,
1981                     struct rte_mempool *mp)
1982 {
1983         struct mrvl_priv *priv = dev->data->dev_private;
1984         struct mrvl_rxq *rxq;
1985         uint32_t frame_size, buf_size = rte_pktmbuf_data_room_size(mp);
1986         uint32_t max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1987         int ret, tc, inq;
1988         uint64_t offloads;
1989
1990         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1991
1992         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1993                 /*
1994                  * Unknown TC mapping, mapping will not have a correct queue.
1995                  */
1996                 MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu",
1997                         idx, priv->ppio_id);
1998                 return -EFAULT;
1999         }
2000
2001         frame_size = buf_size - RTE_PKTMBUF_HEADROOM -
2002                      MRVL_PKT_EFFEC_OFFS + RTE_ETHER_CRC_LEN;
2003         if (frame_size < max_rx_pkt_len) {
2004                 MRVL_LOG(WARNING,
2005                         "Mbuf size must be increased to %u bytes to hold up "
2006                         "to %u bytes of data.",
2007                         buf_size + max_rx_pkt_len - frame_size,
2008                         max_rx_pkt_len);
2009                 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
2010                 MRVL_LOG(INFO, "Setting max rx pkt len to %u",
2011                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
2012         }
2013
2014         if (dev->data->rx_queues[idx]) {
2015                 rte_free(dev->data->rx_queues[idx]);
2016                 dev->data->rx_queues[idx] = NULL;
2017         }
2018
2019         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
2020         if (!rxq)
2021                 return -ENOMEM;
2022
2023         rxq->priv = priv;
2024         rxq->mp = mp;
2025         rxq->cksum_enabled = offloads & DEV_RX_OFFLOAD_IPV4_CKSUM;
2026         rxq->queue_id = idx;
2027         rxq->port_id = dev->data->port_id;
2028         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
2029
2030         tc = priv->rxq_map[rxq->queue_id].tc,
2031         inq = priv->rxq_map[rxq->queue_id].inq;
2032         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
2033                 desc;
2034
2035         ret = mrvl_fill_bpool(rxq, desc);
2036         if (ret) {
2037                 rte_free(rxq);
2038                 return ret;
2039         }
2040
2041         priv->bpool_init_size += desc;
2042
2043         dev->data->rx_queues[idx] = rxq;
2044
2045         return 0;
2046 }
2047
2048 /**
2049  * DPDK callback to release the receive queue.
2050  *
2051  * @param rxq
2052  *   Generic receive queue pointer.
2053  */
2054 static void
2055 mrvl_rx_queue_release(void *rxq)
2056 {
2057         struct mrvl_rxq *q = rxq;
2058         struct pp2_ppio_tc_params *tc_params;
2059         int i, num, tc, inq;
2060         struct pp2_hif *hif;
2061         unsigned int core_id = rte_lcore_id();
2062
2063         if (core_id == LCORE_ID_ANY)
2064                 core_id = rte_get_main_lcore();
2065
2066         if (!q)
2067                 return;
2068
2069         hif = mrvl_get_hif(q->priv, core_id);
2070
2071         if (!hif)
2072                 return;
2073
2074         tc = q->priv->rxq_map[q->queue_id].tc;
2075         inq = q->priv->rxq_map[q->queue_id].inq;
2076         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
2077         num = tc_params->inqs_params[inq].size;
2078         for (i = 0; i < num; i++) {
2079                 struct pp2_buff_inf inf;
2080                 uint64_t addr;
2081
2082                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
2083                 addr = cookie_addr_high | inf.cookie;
2084                 rte_pktmbuf_free((struct rte_mbuf *)addr);
2085         }
2086
2087         rte_free(q);
2088 }
2089
2090 /**
2091  * DPDK callback to configure the transmit queue.
2092  *
2093  * @param dev
2094  *   Pointer to Ethernet device structure.
2095  * @param idx
2096  *   Transmit queue index.
2097  * @param desc
2098  *   Number of descriptors to configure in the queue.
2099  * @param socket
2100  *   NUMA socket on which memory must be allocated.
2101  * @param conf
2102  *   Tx queue configuration parameters.
2103  *
2104  * @return
2105  *   0 on success, negative error value otherwise.
2106  */
2107 static int
2108 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
2109                     unsigned int socket,
2110                     const struct rte_eth_txconf *conf)
2111 {
2112         struct mrvl_priv *priv = dev->data->dev_private;
2113         struct mrvl_txq *txq;
2114
2115         if (dev->data->tx_queues[idx]) {
2116                 rte_free(dev->data->tx_queues[idx]);
2117                 dev->data->tx_queues[idx] = NULL;
2118         }
2119
2120         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
2121         if (!txq)
2122                 return -ENOMEM;
2123
2124         txq->priv = priv;
2125         txq->queue_id = idx;
2126         txq->port_id = dev->data->port_id;
2127         txq->tx_deferred_start = conf->tx_deferred_start;
2128         dev->data->tx_queues[idx] = txq;
2129
2130         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
2131
2132         return 0;
2133 }
2134
2135 /**
2136  * DPDK callback to release the transmit queue.
2137  *
2138  * @param txq
2139  *   Generic transmit queue pointer.
2140  */
2141 static void
2142 mrvl_tx_queue_release(void *txq)
2143 {
2144         struct mrvl_txq *q = txq;
2145
2146         if (!q)
2147                 return;
2148
2149         rte_free(q);
2150 }
2151
2152 /**
2153  * DPDK callback to get flow control configuration.
2154  *
2155  * @param dev
2156  *  Pointer to Ethernet device structure.
2157  * @param fc_conf
2158  *  Pointer to the flow control configuration.
2159  *
2160  * @return
2161  *  0 on success, negative error value otherwise.
2162  */
2163 static int
2164 mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2165 {
2166         struct mrvl_priv *priv = dev->data->dev_private;
2167         int ret, en;
2168
2169         if (!priv->ppio) {
2170                 memcpy(fc_conf, &priv->fc_conf, sizeof(struct rte_eth_fc_conf));
2171                 return 0;
2172         }
2173
2174         fc_conf->autoneg = 1;
2175         ret = pp2_ppio_get_rx_pause(priv->ppio, &en);
2176         if (ret) {
2177                 MRVL_LOG(ERR, "Failed to read rx pause state");
2178                 return ret;
2179         }
2180
2181         fc_conf->mode = en ? RTE_FC_RX_PAUSE : RTE_FC_NONE;
2182
2183         ret = pp2_ppio_get_tx_pause(priv->ppio, &en);
2184         if (ret) {
2185                 MRVL_LOG(ERR, "Failed to read tx pause state");
2186                 return ret;
2187         }
2188
2189         if (en) {
2190                 if (fc_conf->mode == RTE_FC_NONE)
2191                         fc_conf->mode = RTE_FC_TX_PAUSE;
2192                 else
2193                         fc_conf->mode = RTE_FC_FULL;
2194         }
2195
2196         return 0;
2197 }
2198
2199 /**
2200  * DPDK callback to set flow control configuration.
2201  *
2202  * @param dev
2203  *  Pointer to Ethernet device structure.
2204  * @param fc_conf
2205  *  Pointer to the flow control configuration.
2206  *
2207  * @return
2208  *  0 on success, negative error value otherwise.
2209  */
2210 static int
2211 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2212 {
2213         struct mrvl_priv *priv = dev->data->dev_private;
2214         struct pp2_ppio_tx_pause_params mrvl_pause_params;
2215         int ret;
2216         int rx_en, tx_en;
2217
2218         if (fc_conf->high_water ||
2219             fc_conf->low_water ||
2220             fc_conf->pause_time ||
2221             fc_conf->mac_ctrl_frame_fwd) {
2222                 MRVL_LOG(ERR, "Flowctrl parameter is not supported");
2223
2224                 return -EINVAL;
2225         }
2226
2227         if (fc_conf->autoneg == 0) {
2228                 MRVL_LOG(ERR, "Flowctrl Autoneg disable is not supported");
2229                 return -EINVAL;
2230         }
2231
2232         if (!priv->ppio) {
2233                 memcpy(&priv->fc_conf, fc_conf, sizeof(struct rte_eth_fc_conf));
2234                 priv->flow_ctrl = 1;
2235                 return 0;
2236         }
2237
2238         switch (fc_conf->mode) {
2239         case RTE_FC_FULL:
2240                 rx_en = 1;
2241                 tx_en = 1;
2242                 break;
2243         case RTE_FC_TX_PAUSE:
2244                 rx_en = 0;
2245                 tx_en = 1;
2246                 break;
2247         case RTE_FC_RX_PAUSE:
2248                 rx_en = 1;
2249                 tx_en = 0;
2250                 break;
2251         case RTE_FC_NONE:
2252                 rx_en = 0;
2253                 tx_en = 0;
2254                 break;
2255         default:
2256                 MRVL_LOG(ERR, "Incorrect Flow control flag (%d)",
2257                          fc_conf->mode);
2258                 return -EINVAL;
2259         }
2260
2261         /* Set RX flow control */
2262         ret = pp2_ppio_set_rx_pause(priv->ppio, rx_en);
2263         if (ret) {
2264                 MRVL_LOG(ERR, "Failed to change RX flowctrl");
2265                 return ret;
2266         }
2267
2268         /* Set TX flow control */
2269         mrvl_pause_params.en = tx_en;
2270         /* all inqs participate in xon/xoff decision */
2271         mrvl_pause_params.use_tc_pause_inqs = 0;
2272         ret = pp2_ppio_set_tx_pause(priv->ppio, &mrvl_pause_params);
2273         if (ret) {
2274                 MRVL_LOG(ERR, "Failed to change TX flowctrl");
2275                 return ret;
2276         }
2277
2278         return 0;
2279 }
2280
2281 /**
2282  * Update RSS hash configuration
2283  *
2284  * @param dev
2285  *   Pointer to Ethernet device structure.
2286  * @param rss_conf
2287  *   Pointer to RSS configuration.
2288  *
2289  * @return
2290  *   0 on success, negative error value otherwise.
2291  */
2292 static int
2293 mrvl_rss_hash_update(struct rte_eth_dev *dev,
2294                      struct rte_eth_rss_conf *rss_conf)
2295 {
2296         struct mrvl_priv *priv = dev->data->dev_private;
2297
2298         if (priv->isolated)
2299                 return -ENOTSUP;
2300
2301         return mrvl_configure_rss(priv, rss_conf);
2302 }
2303
2304 /**
2305  * DPDK callback to get RSS hash configuration.
2306  *
2307  * @param dev
2308  *   Pointer to Ethernet device structure.
2309  * @rss_conf
2310  *   Pointer to RSS configuration.
2311  *
2312  * @return
2313  *   Always 0.
2314  */
2315 static int
2316 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
2317                        struct rte_eth_rss_conf *rss_conf)
2318 {
2319         struct mrvl_priv *priv = dev->data->dev_private;
2320         enum pp2_ppio_hash_type hash_type =
2321                 priv->ppio_params.inqs_params.hash_type;
2322
2323         rss_conf->rss_key = NULL;
2324
2325         if (hash_type == PP2_PPIO_HASH_T_NONE)
2326                 rss_conf->rss_hf = 0;
2327         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
2328                 rss_conf->rss_hf = ETH_RSS_IPV4;
2329         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
2330                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
2331         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
2332                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
2333
2334         return 0;
2335 }
2336
2337 /**
2338  * DPDK callback to get rte_flow callbacks.
2339  *
2340  * @param dev
2341  *   Pointer to the device structure.
2342  * @param filer_type
2343  *   Flow filter type.
2344  * @param filter_op
2345  *   Flow filter operation.
2346  * @param arg
2347  *   Pointer to pass the flow ops.
2348  *
2349  * @return
2350  *   0 on success, negative error value otherwise.
2351  */
2352 static int
2353 mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
2354                      enum rte_filter_type filter_type,
2355                      enum rte_filter_op filter_op, void *arg)
2356 {
2357         switch (filter_type) {
2358         case RTE_ETH_FILTER_GENERIC:
2359                 if (filter_op != RTE_ETH_FILTER_GET)
2360                         return -EINVAL;
2361                 *(const void **)arg = &mrvl_flow_ops;
2362                 return 0;
2363         default:
2364                 MRVL_LOG(WARNING, "Filter type (%d) not supported",
2365                                 filter_type);
2366                 return -EINVAL;
2367         }
2368 }
2369
2370 /**
2371  * DPDK callback to get rte_mtr callbacks.
2372  *
2373  * @param dev
2374  *   Pointer to the device structure.
2375  * @param ops
2376  *   Pointer to pass the mtr ops.
2377  *
2378  * @return
2379  *   Always 0.
2380  */
2381 static int
2382 mrvl_mtr_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
2383 {
2384         *(const void **)ops = &mrvl_mtr_ops;
2385
2386         return 0;
2387 }
2388
2389 /**
2390  * DPDK callback to get rte_tm callbacks.
2391  *
2392  * @param dev
2393  *   Pointer to the device structure.
2394  * @param ops
2395  *   Pointer to pass the tm ops.
2396  *
2397  * @return
2398  *   Always 0.
2399  */
2400 static int
2401 mrvl_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
2402 {
2403         *(const void **)ops = &mrvl_tm_ops;
2404
2405         return 0;
2406 }
2407
2408 static const struct eth_dev_ops mrvl_ops = {
2409         .dev_configure = mrvl_dev_configure,
2410         .dev_start = mrvl_dev_start,
2411         .dev_stop = mrvl_dev_stop,
2412         .dev_set_link_up = mrvl_dev_set_link_up,
2413         .dev_set_link_down = mrvl_dev_set_link_down,
2414         .dev_close = mrvl_dev_close,
2415         .link_update = mrvl_link_update,
2416         .promiscuous_enable = mrvl_promiscuous_enable,
2417         .allmulticast_enable = mrvl_allmulticast_enable,
2418         .promiscuous_disable = mrvl_promiscuous_disable,
2419         .allmulticast_disable = mrvl_allmulticast_disable,
2420         .mac_addr_remove = mrvl_mac_addr_remove,
2421         .mac_addr_add = mrvl_mac_addr_add,
2422         .mac_addr_set = mrvl_mac_addr_set,
2423         .mtu_set = mrvl_mtu_set,
2424         .stats_get = mrvl_stats_get,
2425         .stats_reset = mrvl_stats_reset,
2426         .xstats_get = mrvl_xstats_get,
2427         .xstats_reset = mrvl_xstats_reset,
2428         .xstats_get_names = mrvl_xstats_get_names,
2429         .dev_infos_get = mrvl_dev_infos_get,
2430         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
2431         .rxq_info_get = mrvl_rxq_info_get,
2432         .txq_info_get = mrvl_txq_info_get,
2433         .vlan_filter_set = mrvl_vlan_filter_set,
2434         .vlan_offload_set = mrvl_vlan_offload_set,
2435         .tx_queue_start = mrvl_tx_queue_start,
2436         .tx_queue_stop = mrvl_tx_queue_stop,
2437         .rx_queue_setup = mrvl_rx_queue_setup,
2438         .rx_queue_release = mrvl_rx_queue_release,
2439         .tx_queue_setup = mrvl_tx_queue_setup,
2440         .tx_queue_release = mrvl_tx_queue_release,
2441         .flow_ctrl_get = mrvl_flow_ctrl_get,
2442         .flow_ctrl_set = mrvl_flow_ctrl_set,
2443         .rss_hash_update = mrvl_rss_hash_update,
2444         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
2445         .filter_ctrl = mrvl_eth_filter_ctrl,
2446         .mtr_ops_get = mrvl_mtr_ops_get,
2447         .tm_ops_get = mrvl_tm_ops_get,
2448 };
2449
2450 /**
2451  * Return packet type information and l3/l4 offsets.
2452  *
2453  * @param desc
2454  *   Pointer to the received packet descriptor.
2455  * @param l3_offset
2456  *   l3 packet offset.
2457  * @param l4_offset
2458  *   l4 packet offset.
2459  *
2460  * @return
2461  *   Packet type information.
2462  */
2463 static inline uint64_t
2464 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
2465                                     uint8_t *l3_offset, uint8_t *l4_offset)
2466 {
2467         enum pp2_inq_l3_type l3_type;
2468         enum pp2_inq_l4_type l4_type;
2469         enum pp2_inq_vlan_tag vlan_tag;
2470         uint64_t packet_type;
2471
2472         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
2473         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
2474         pp2_ppio_inq_desc_get_vlan_tag(desc, &vlan_tag);
2475
2476         packet_type = RTE_PTYPE_L2_ETHER;
2477
2478         switch (vlan_tag) {
2479         case PP2_INQ_VLAN_TAG_SINGLE:
2480                 packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
2481                 break;
2482         case PP2_INQ_VLAN_TAG_DOUBLE:
2483         case PP2_INQ_VLAN_TAG_TRIPLE:
2484                 packet_type |= RTE_PTYPE_L2_ETHER_QINQ;
2485                 break;
2486         default:
2487                 break;
2488         }
2489
2490         switch (l3_type) {
2491         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
2492                 packet_type |= RTE_PTYPE_L3_IPV4;
2493                 break;
2494         case PP2_INQ_L3_TYPE_IPV4_OK:
2495                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
2496                 break;
2497         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
2498                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
2499                 break;
2500         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
2501                 packet_type |= RTE_PTYPE_L3_IPV6;
2502                 break;
2503         case PP2_INQ_L3_TYPE_IPV6_EXT:
2504                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
2505                 break;
2506         case PP2_INQ_L3_TYPE_ARP:
2507                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
2508                 /*
2509                  * In case of ARP l4_offset is set to wrong value.
2510                  * Set it to proper one so that later on mbuf->l3_len can be
2511                  * calculated subtracting l4_offset and l3_offset.
2512                  */
2513                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
2514                 break;
2515         default:
2516                 break;
2517         }
2518
2519         switch (l4_type) {
2520         case PP2_INQ_L4_TYPE_TCP:
2521                 packet_type |= RTE_PTYPE_L4_TCP;
2522                 break;
2523         case PP2_INQ_L4_TYPE_UDP:
2524                 packet_type |= RTE_PTYPE_L4_UDP;
2525                 break;
2526         default:
2527                 break;
2528         }
2529
2530         return packet_type;
2531 }
2532
2533 /**
2534  * Get offload information from the received packet descriptor.
2535  *
2536  * @param desc
2537  *   Pointer to the received packet descriptor.
2538  *
2539  * @return
2540  *   Mbuf offload flags.
2541  */
2542 static inline uint64_t
2543 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc, uint64_t packet_type)
2544 {
2545         uint64_t flags = 0;
2546         enum pp2_inq_desc_status status;
2547
2548         if (RTE_ETH_IS_IPV4_HDR(packet_type)) {
2549                 status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
2550                 if (unlikely(status != PP2_DESC_ERR_OK))
2551                         flags |= PKT_RX_IP_CKSUM_BAD;
2552                 else
2553                         flags |= PKT_RX_IP_CKSUM_GOOD;
2554         }
2555
2556         if (((packet_type & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) ||
2557             ((packet_type & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP)) {
2558                 status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
2559                 if (unlikely(status != PP2_DESC_ERR_OK))
2560                         flags |= PKT_RX_L4_CKSUM_BAD;
2561                 else
2562                         flags |= PKT_RX_L4_CKSUM_GOOD;
2563         }
2564
2565         return flags;
2566 }
2567
2568 /**
2569  * DPDK callback for receive.
2570  *
2571  * @param rxq
2572  *   Generic pointer to the receive queue.
2573  * @param rx_pkts
2574  *   Array to store received packets.
2575  * @param nb_pkts
2576  *   Maximum number of packets in array.
2577  *
2578  * @return
2579  *   Number of packets successfully received.
2580  */
2581 static uint16_t
2582 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2583 {
2584         struct mrvl_rxq *q = rxq;
2585         struct pp2_ppio_desc descs[nb_pkts];
2586         struct pp2_bpool *bpool;
2587         int i, ret, rx_done = 0;
2588         int num;
2589         struct pp2_hif *hif;
2590         unsigned int core_id = rte_lcore_id();
2591
2592         hif = mrvl_get_hif(q->priv, core_id);
2593
2594         if (unlikely(!q->priv->ppio || !hif))
2595                 return 0;
2596
2597         bpool = q->priv->bpool;
2598
2599         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
2600                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
2601         if (unlikely(ret < 0))
2602                 return 0;
2603
2604         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
2605
2606         for (i = 0; i < nb_pkts; i++) {
2607                 struct rte_mbuf *mbuf;
2608                 uint8_t l3_offset, l4_offset;
2609                 enum pp2_inq_desc_status status;
2610                 uint64_t addr;
2611
2612                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2613                         struct pp2_ppio_desc *pref_desc;
2614                         u64 pref_addr;
2615
2616                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
2617                         pref_addr = cookie_addr_high |
2618                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
2619                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
2620                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
2621                 }
2622
2623                 addr = cookie_addr_high |
2624                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
2625                 mbuf = (struct rte_mbuf *)addr;
2626                 rte_pktmbuf_reset(mbuf);
2627
2628                 /* drop packet in case of mac, overrun or resource error */
2629                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
2630                 if ((unlikely(status != PP2_DESC_ERR_OK)) &&
2631                         !(q->priv->forward_bad_frames)) {
2632                         struct pp2_buff_inf binf = {
2633                                 .addr = rte_mbuf_data_iova_default(mbuf),
2634                                 .cookie = (uint64_t)mbuf,
2635                         };
2636
2637                         pp2_bpool_put_buff(hif, bpool, &binf);
2638                         mrvl_port_bpool_size
2639                                 [bpool->pp2_id][bpool->id][core_id]++;
2640                         q->drop_mac++;
2641                         continue;
2642                 }
2643
2644                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
2645                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
2646                 mbuf->data_len = mbuf->pkt_len;
2647                 mbuf->port = q->port_id;
2648                 mbuf->packet_type =
2649                         mrvl_desc_to_packet_type_and_offset(&descs[i],
2650                                                             &l3_offset,
2651                                                             &l4_offset);
2652                 mbuf->l2_len = l3_offset;
2653                 mbuf->l3_len = l4_offset - l3_offset;
2654
2655                 if (likely(q->cksum_enabled))
2656                         mbuf->ol_flags =
2657                                 mrvl_desc_to_ol_flags(&descs[i],
2658                                                       mbuf->packet_type);
2659
2660                 rx_pkts[rx_done++] = mbuf;
2661                 q->bytes_recv += mbuf->pkt_len;
2662         }
2663
2664         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
2665                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
2666
2667                 if (unlikely(num <= q->priv->bpool_min_size ||
2668                              (!rx_done && num < q->priv->bpool_init_size))) {
2669                         mrvl_fill_bpool(q, MRVL_BURST_SIZE);
2670                 } else if (unlikely(num > q->priv->bpool_max_size)) {
2671                         int i;
2672                         int pkt_to_remove = num - q->priv->bpool_init_size;
2673                         struct rte_mbuf *mbuf;
2674                         struct pp2_buff_inf buff;
2675
2676                         for (i = 0; i < pkt_to_remove; i++) {
2677                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
2678                                 if (ret)
2679                                         break;
2680                                 mbuf = (struct rte_mbuf *)
2681                                         (cookie_addr_high | buff.cookie);
2682                                 rte_pktmbuf_free(mbuf);
2683                         }
2684                         mrvl_port_bpool_size
2685                                 [bpool->pp2_id][bpool->id][core_id] -= i;
2686                 }
2687                 rte_spinlock_unlock(&q->priv->lock);
2688         }
2689
2690         return rx_done;
2691 }
2692
2693 /**
2694  * Prepare offload information.
2695  *
2696  * @param ol_flags
2697  *   Offload flags.
2698  * @param l3_type
2699  *   Pointer to the pp2_ouq_l3_type structure.
2700  * @param l4_type
2701  *   Pointer to the pp2_outq_l4_type structure.
2702  * @param gen_l3_cksum
2703  *   Will be set to 1 in case l3 checksum is computed.
2704  * @param l4_cksum
2705  *   Will be set to 1 in case l4 checksum is computed.
2706  */
2707 static inline void
2708 mrvl_prepare_proto_info(uint64_t ol_flags,
2709                         enum pp2_outq_l3_type *l3_type,
2710                         enum pp2_outq_l4_type *l4_type,
2711                         int *gen_l3_cksum,
2712                         int *gen_l4_cksum)
2713 {
2714         /*
2715          * Based on ol_flags prepare information
2716          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
2717          * for offloading.
2718          * in most of the checksum cases ipv4 must be set, so this is the
2719          * default value
2720          */
2721         *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
2722         *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
2723
2724         if (ol_flags & PKT_TX_IPV6) {
2725                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
2726                 /* no checksum for ipv6 header */
2727                 *gen_l3_cksum = 0;
2728         }
2729
2730         if ((ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) {
2731                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
2732                 *gen_l4_cksum = 1;
2733         } else if ((ol_flags & PKT_TX_L4_MASK) ==  PKT_TX_UDP_CKSUM) {
2734                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
2735                 *gen_l4_cksum = 1;
2736         } else {
2737                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
2738                 /* no checksum for other type */
2739                 *gen_l4_cksum = 0;
2740         }
2741 }
2742
2743 /**
2744  * Release already sent buffers to bpool (buffer-pool).
2745  *
2746  * @param ppio
2747  *   Pointer to the port structure.
2748  * @param hif
2749  *   Pointer to the MUSDK hardware interface.
2750  * @param sq
2751  *   Pointer to the shadow queue.
2752  * @param qid
2753  *   Queue id number.
2754  * @param force
2755  *   Force releasing packets.
2756  */
2757 static inline void
2758 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
2759                        unsigned int core_id, struct mrvl_shadow_txq *sq,
2760                        int qid, int force)
2761 {
2762         struct buff_release_entry *entry;
2763         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
2764         int i;
2765
2766         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
2767
2768         sq->num_to_release += nb_done;
2769
2770         if (likely(!force &&
2771                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
2772                 return;
2773
2774         nb_done = sq->num_to_release;
2775         sq->num_to_release = 0;
2776
2777         for (i = 0; i < nb_done; i++) {
2778                 entry = &sq->ent[sq->tail + num];
2779                 if (unlikely(!entry->buff.addr)) {
2780                         MRVL_LOG(ERR,
2781                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!",
2782                                 sq->tail, (u64)entry->buff.cookie,
2783                                 (u64)entry->buff.addr);
2784                         skip_bufs = 1;
2785                         goto skip;
2786                 }
2787
2788                 if (unlikely(!entry->bpool)) {
2789                         struct rte_mbuf *mbuf;
2790
2791                         mbuf = (struct rte_mbuf *)entry->buff.cookie;
2792                         rte_pktmbuf_free(mbuf);
2793                         skip_bufs = 1;
2794                         goto skip;
2795                 }
2796
2797                 mrvl_port_bpool_size
2798                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
2799                 num++;
2800                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
2801                         goto skip;
2802                 continue;
2803 skip:
2804                 if (likely(num))
2805                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2806                 num += skip_bufs;
2807                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2808                 sq->size -= num;
2809                 num = 0;
2810                 skip_bufs = 0;
2811         }
2812
2813         if (likely(num)) {
2814                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2815                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2816                 sq->size -= num;
2817         }
2818 }
2819
2820 /**
2821  * DPDK callback for transmit.
2822  *
2823  * @param txq
2824  *   Generic pointer transmit queue.
2825  * @param tx_pkts
2826  *   Packets to transmit.
2827  * @param nb_pkts
2828  *   Number of packets in array.
2829  *
2830  * @return
2831  *   Number of packets successfully transmitted.
2832  */
2833 static uint16_t
2834 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2835 {
2836         struct mrvl_txq *q = txq;
2837         struct mrvl_shadow_txq *sq;
2838         struct pp2_hif *hif;
2839         struct pp2_ppio_desc descs[nb_pkts];
2840         unsigned int core_id = rte_lcore_id();
2841         int i, bytes_sent = 0;
2842         uint16_t num, sq_free_size;
2843         uint64_t addr;
2844
2845         hif = mrvl_get_hif(q->priv, core_id);
2846         sq = &q->shadow_txqs[core_id];
2847
2848         if (unlikely(!q->priv->ppio || !hif))
2849                 return 0;
2850
2851         if (sq->size)
2852                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2853                                        sq, q->queue_id, 0);
2854
2855         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2856         if (unlikely(nb_pkts > sq_free_size))
2857                 nb_pkts = sq_free_size;
2858
2859         for (i = 0; i < nb_pkts; i++) {
2860                 struct rte_mbuf *mbuf = tx_pkts[i];
2861                 int gen_l3_cksum, gen_l4_cksum;
2862                 enum pp2_outq_l3_type l3_type;
2863                 enum pp2_outq_l4_type l4_type;
2864
2865                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2866                         struct rte_mbuf *pref_pkt_hdr;
2867
2868                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2869                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2870                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2871                 }
2872
2873                 mrvl_fill_shadowq(sq, mbuf);
2874                 mrvl_fill_desc(&descs[i], mbuf);
2875
2876                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2877                 /*
2878                  * in case unsupported ol_flags were passed
2879                  * do not update descriptor offload information
2880                  */
2881                 if (!(mbuf->ol_flags & MRVL_TX_PKT_OFFLOADS))
2882                         continue;
2883                 mrvl_prepare_proto_info(mbuf->ol_flags, &l3_type, &l4_type,
2884                                         &gen_l3_cksum, &gen_l4_cksum);
2885
2886                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2887                                                   mbuf->l2_len,
2888                                                   mbuf->l2_len + mbuf->l3_len,
2889                                                   gen_l3_cksum, gen_l4_cksum);
2890         }
2891
2892         num = nb_pkts;
2893         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2894         /* number of packets that were not sent */
2895         if (unlikely(num > nb_pkts)) {
2896                 for (i = nb_pkts; i < num; i++) {
2897                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2898                                 MRVL_PP2_TX_SHADOWQ_MASK;
2899                         addr = sq->ent[sq->head].buff.cookie;
2900                         bytes_sent -=
2901                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2902                 }
2903                 sq->size -= num - nb_pkts;
2904         }
2905
2906         q->bytes_sent += bytes_sent;
2907
2908         return nb_pkts;
2909 }
2910
2911 /** DPDK callback for S/G transmit.
2912  *
2913  * @param txq
2914  *   Generic pointer transmit queue.
2915  * @param tx_pkts
2916  *   Packets to transmit.
2917  * @param nb_pkts
2918  *   Number of packets in array.
2919  *
2920  * @return
2921  *   Number of packets successfully transmitted.
2922  */
2923 static uint16_t
2924 mrvl_tx_sg_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
2925                      uint16_t nb_pkts)
2926 {
2927         struct mrvl_txq *q = txq;
2928         struct mrvl_shadow_txq *sq;
2929         struct pp2_hif *hif;
2930         struct pp2_ppio_desc descs[nb_pkts * PP2_PPIO_DESC_NUM_FRAGS];
2931         struct pp2_ppio_sg_pkts pkts;
2932         uint8_t frags[nb_pkts];
2933         unsigned int core_id = rte_lcore_id();
2934         int i, j, bytes_sent = 0;
2935         int tail, tail_first;
2936         uint16_t num, sq_free_size;
2937         uint16_t nb_segs, total_descs = 0;
2938         uint64_t addr;
2939
2940         hif = mrvl_get_hif(q->priv, core_id);
2941         sq = &q->shadow_txqs[core_id];
2942         pkts.frags = frags;
2943         pkts.num = 0;
2944
2945         if (unlikely(!q->priv->ppio || !hif))
2946                 return 0;
2947
2948         if (sq->size)
2949                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2950                                        sq, q->queue_id, 0);
2951
2952         /* Save shadow queue free size */
2953         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2954
2955         tail = 0;
2956         for (i = 0; i < nb_pkts; i++) {
2957                 struct rte_mbuf *mbuf = tx_pkts[i];
2958                 struct rte_mbuf *seg = NULL;
2959                 int gen_l3_cksum, gen_l4_cksum;
2960                 enum pp2_outq_l3_type l3_type;
2961                 enum pp2_outq_l4_type l4_type;
2962
2963                 nb_segs = mbuf->nb_segs;
2964                 tail_first = tail;
2965                 total_descs += nb_segs;
2966
2967                 /*
2968                  * Check if total_descs does not exceed
2969                  * shadow queue free size
2970                  */
2971                 if (unlikely(total_descs > sq_free_size)) {
2972                         total_descs -= nb_segs;
2973                         break;
2974                 }
2975
2976                 /* Check if nb_segs does not exceed the max nb of desc per
2977                  * fragmented packet
2978                  */
2979                 if (nb_segs > PP2_PPIO_DESC_NUM_FRAGS) {
2980                         total_descs -= nb_segs;
2981                         RTE_LOG(ERR, PMD,
2982                                 "Too many segments. Packet won't be sent.\n");
2983                         break;
2984                 }
2985
2986                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2987                         struct rte_mbuf *pref_pkt_hdr;
2988
2989                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2990                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2991                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2992                 }
2993
2994                 pkts.frags[pkts.num] = nb_segs;
2995                 pkts.num++;
2996
2997                 seg = mbuf;
2998                 for (j = 0; j < nb_segs - 1; j++) {
2999                         /* For the subsequent segments, set shadow queue
3000                          * buffer to NULL
3001                          */
3002                         mrvl_fill_shadowq(sq, NULL);
3003                         mrvl_fill_desc(&descs[tail], seg);
3004
3005                         tail++;
3006                         seg = seg->next;
3007                 }
3008                 /* Put first mbuf info in last shadow queue entry */
3009                 mrvl_fill_shadowq(sq, mbuf);
3010                 /* Update descriptor with last segment */
3011                 mrvl_fill_desc(&descs[tail++], seg);
3012
3013                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
3014                 /* In case unsupported ol_flags were passed
3015                  * do not update descriptor offload information
3016                  */
3017                 if (!(mbuf->ol_flags & MRVL_TX_PKT_OFFLOADS))
3018                         continue;
3019                 mrvl_prepare_proto_info(mbuf->ol_flags, &l3_type, &l4_type,
3020                                         &gen_l3_cksum, &gen_l4_cksum);
3021
3022                 pp2_ppio_outq_desc_set_proto_info(&descs[tail_first], l3_type,
3023                                                   l4_type, mbuf->l2_len,
3024                                                   mbuf->l2_len + mbuf->l3_len,
3025                                                   gen_l3_cksum, gen_l4_cksum);
3026         }
3027
3028         num = total_descs;
3029         pp2_ppio_send_sg(q->priv->ppio, hif, q->queue_id, descs,
3030                          &total_descs, &pkts);
3031         /* number of packets that were not sent */
3032         if (unlikely(num > total_descs)) {
3033                 for (i = total_descs; i < num; i++) {
3034                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
3035                                 MRVL_PP2_TX_SHADOWQ_MASK;
3036
3037                         addr = sq->ent[sq->head].buff.cookie;
3038                         if (addr)
3039                                 bytes_sent -=
3040                                         rte_pktmbuf_pkt_len((struct rte_mbuf *)
3041                                                 (cookie_addr_high | addr));
3042                 }
3043                 sq->size -= num - total_descs;
3044                 nb_pkts = pkts.num;
3045         }
3046
3047         q->bytes_sent += bytes_sent;
3048
3049         return nb_pkts;
3050 }
3051
3052 /**
3053  * Create private device structure.
3054  *
3055  * @param dev_name
3056  *   Pointer to the port name passed in the initialization parameters.
3057  *
3058  * @return
3059  *   Pointer to the newly allocated private device structure.
3060  */
3061 static struct mrvl_priv *
3062 mrvl_priv_create(const char *dev_name)
3063 {
3064         struct pp2_bpool_params bpool_params;
3065         char match[MRVL_MATCH_LEN];
3066         struct mrvl_priv *priv;
3067         uint16_t max_frame_size;
3068         int ret, bpool_bit;
3069
3070         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
3071         if (!priv)
3072                 return NULL;
3073
3074         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
3075                                        &priv->pp_id, &priv->ppio_id);
3076         if (ret)
3077                 goto out_free_priv;
3078
3079         ret = pp2_ppio_get_l4_cksum_max_frame_size(priv->pp_id, priv->ppio_id,
3080                                                    &max_frame_size);
3081         if (ret)
3082                 goto out_free_priv;
3083
3084         priv->max_mtu = max_frame_size + RTE_ETHER_CRC_LEN -
3085                 MRVL_PP2_ETH_HDRS_LEN;
3086
3087         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
3088                                      PP2_BPOOL_NUM_POOLS);
3089         if (bpool_bit < 0)
3090                 goto out_free_priv;
3091         priv->bpool_bit = bpool_bit;
3092
3093         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
3094                  priv->bpool_bit);
3095         memset(&bpool_params, 0, sizeof(bpool_params));
3096         bpool_params.match = match;
3097         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
3098         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
3099         if (ret)
3100                 goto out_clear_bpool_bit;
3101
3102         priv->ppio_params.type = PP2_PPIO_T_NIC;
3103         rte_spinlock_init(&priv->lock);
3104
3105         return priv;
3106 out_clear_bpool_bit:
3107         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
3108 out_free_priv:
3109         rte_free(priv);
3110         return NULL;
3111 }
3112
3113 /**
3114  * Create device representing Ethernet port.
3115  *
3116  * @param name
3117  *   Pointer to the port's name.
3118  *
3119  * @return
3120  *   0 on success, negative error value otherwise.
3121  */
3122 static int
3123 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
3124 {
3125         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
3126         struct rte_eth_dev *eth_dev;
3127         struct mrvl_priv *priv;
3128         struct ifreq req;
3129
3130         eth_dev = rte_eth_dev_allocate(name);
3131         if (!eth_dev)
3132                 return -ENOMEM;
3133
3134         priv = mrvl_priv_create(name);
3135         if (!priv) {
3136                 ret = -ENOMEM;
3137                 goto out_free;
3138         }
3139         eth_dev->data->dev_private = priv;
3140
3141         eth_dev->data->mac_addrs =
3142                 rte_zmalloc("mac_addrs",
3143                             RTE_ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
3144         if (!eth_dev->data->mac_addrs) {
3145                 MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
3146                 ret = -ENOMEM;
3147                 goto out_free;
3148         }
3149
3150         memset(&req, 0, sizeof(req));
3151         strcpy(req.ifr_name, name);
3152         ret = ioctl(fd, SIOCGIFHWADDR, &req);
3153         if (ret)
3154                 goto out_free;
3155
3156         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
3157                req.ifr_addr.sa_data, RTE_ETHER_ADDR_LEN);
3158
3159         eth_dev->device = &vdev->device;
3160         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
3161         mrvl_set_tx_function(eth_dev);
3162         eth_dev->dev_ops = &mrvl_ops;
3163         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
3164
3165         eth_dev->data->dev_link.link_status = ETH_LINK_UP;
3166
3167         rte_eth_dev_probing_finish(eth_dev);
3168         return 0;
3169 out_free:
3170         rte_eth_dev_release_port(eth_dev);
3171
3172         return ret;
3173 }
3174
3175 /**
3176  * Callback used by rte_kvargs_process() during argument parsing.
3177  *
3178  * @param key
3179  *   Pointer to the parsed key (unused).
3180  * @param value
3181  *   Pointer to the parsed value.
3182  * @param extra_args
3183  *   Pointer to the extra arguments which contains address of the
3184  *   table of pointers to parsed interface names.
3185  *
3186  * @return
3187  *   Always 0.
3188  */
3189 static int
3190 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
3191                  void *extra_args)
3192 {
3193         struct mrvl_ifnames *ifnames = extra_args;
3194
3195         ifnames->names[ifnames->idx++] = value;
3196
3197         return 0;
3198 }
3199
3200 /**
3201  * DPDK callback to register the virtual device.
3202  *
3203  * @param vdev
3204  *   Pointer to the virtual device.
3205  *
3206  * @return
3207  *   0 on success, negative error value otherwise.
3208  */
3209 static int
3210 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
3211 {
3212         struct rte_kvargs *kvlist;
3213         struct mrvl_ifnames ifnames;
3214         int ret = -EINVAL;
3215         uint32_t i, ifnum, cfgnum;
3216         const char *params;
3217
3218         params = rte_vdev_device_args(vdev);
3219         if (!params)
3220                 return -EINVAL;
3221
3222         kvlist = rte_kvargs_parse(params, valid_args);
3223         if (!kvlist)
3224                 return -EINVAL;
3225
3226         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
3227         if (ifnum > RTE_DIM(ifnames.names))
3228                 goto out_free_kvlist;
3229
3230         ifnames.idx = 0;
3231         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
3232                            mrvl_get_ifnames, &ifnames);
3233
3234
3235         /*
3236          * The below system initialization should be done only once,
3237          * on the first provided configuration file
3238          */
3239         if (!mrvl_cfg) {
3240                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
3241                 MRVL_LOG(INFO, "Parsing config file!");
3242                 if (cfgnum > 1) {
3243                         MRVL_LOG(ERR, "Cannot handle more than one config file!");
3244                         goto out_free_kvlist;
3245                 } else if (cfgnum == 1) {
3246                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
3247                                            mrvl_get_cfg, &mrvl_cfg);
3248                 }
3249         }
3250
3251         if (mrvl_dev_num)
3252                 goto init_devices;
3253
3254         MRVL_LOG(INFO, "Perform MUSDK initializations");
3255
3256         ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist);
3257         if (ret)
3258                 goto out_free_kvlist;
3259
3260         ret = mrvl_init_pp2();
3261         if (ret) {
3262                 MRVL_LOG(ERR, "Failed to init PP!");
3263                 rte_mvep_deinit(MVEP_MOD_T_PP2);
3264                 goto out_free_kvlist;
3265         }
3266
3267         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
3268         memset(mrvl_port_to_bpool_lookup, 0, sizeof(mrvl_port_to_bpool_lookup));
3269
3270         mrvl_lcore_first = RTE_MAX_LCORE;
3271         mrvl_lcore_last = 0;
3272
3273 init_devices:
3274         for (i = 0; i < ifnum; i++) {
3275                 MRVL_LOG(INFO, "Creating %s", ifnames.names[i]);
3276                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
3277                 if (ret)
3278                         goto out_cleanup;
3279                 mrvl_dev_num++;
3280         }
3281
3282         rte_kvargs_free(kvlist);
3283
3284         return 0;
3285 out_cleanup:
3286         rte_pmd_mrvl_remove(vdev);
3287
3288 out_free_kvlist:
3289         rte_kvargs_free(kvlist);
3290
3291         return ret;
3292 }
3293
3294 /**
3295  * DPDK callback to remove virtual device.
3296  *
3297  * @param vdev
3298  *   Pointer to the removed virtual device.
3299  *
3300  * @return
3301  *   0 on success, negative error value otherwise.
3302  */
3303 static int
3304 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
3305 {
3306         uint16_t port_id;
3307         int ret = 0;
3308
3309         RTE_ETH_FOREACH_DEV(port_id) {
3310                 if (rte_eth_devices[port_id].device != &vdev->device)
3311                         continue;
3312                 ret |= rte_eth_dev_close(port_id);
3313         }
3314
3315         return ret == 0 ? 0 : -EIO;
3316 }
3317
3318 static struct rte_vdev_driver pmd_mrvl_drv = {
3319         .probe = rte_pmd_mrvl_probe,
3320         .remove = rte_pmd_mrvl_remove,
3321 };
3322
3323 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv);
3324 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2);
3325 RTE_LOG_REGISTER(mrvl_logtype, pmd.net.mvpp2, NOTICE);