drivers/net/mvpp2/mrvl_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Marvell International Ltd.
   3  * Copyright(c) 2017 Semihalf.
   4  * All rights reserved.
   5  */
   6
   7 #include <rte_ethdev_driver.h>
   8 #include <rte_kvargs.h>
   9 #include <rte_log.h>
  10 #include <rte_malloc.h>
  11 #include <rte_bus_vdev.h>
  12
  13 /* Unluckily, container_of is defined by both DPDK and MUSDK,
  14  * we'll declare only one version.
  15  *
  16  * Note that it is not used in this PMD anyway.
  17  */
  18 #ifdef container_of
  19 #undef container_of
  20 #endif
  21
  22 #include <fcntl.h>
  23 #include <linux/ethtool.h>
  24 #include <linux/sockios.h>
  25 #include <net/if.h>
  26 #include <net/if_arp.h>
  27 #include <sys/ioctl.h>
  28 #include <sys/socket.h>
  29 #include <sys/stat.h>
  30 #include <sys/types.h>
  31
  32 #include <rte_mvep_common.h>
  33 #include "mrvl_ethdev.h"
  34 #include "mrvl_qos.h"
  35
  36 /* bitmask with reserved hifs */
  37 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  38 /* bitmask with reserved bpools */
  39 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  40 /* bitmask with reserved kernel RSS tables */
  41 #define MRVL_MUSDK_RSS_RESERVED 0x01
  42 /* maximum number of available hifs */
  43 #define MRVL_MUSDK_HIFS_MAX 9
  44
  45 /* prefetch shift */
  46 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  47
  48 /* TCAM has 25 entries reserved for uc/mc filter entries */
  49 #define MRVL_MAC_ADDRS_MAX 25
  50 #define MRVL_MATCH_LEN 16
  51 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  52 /* Maximum allowable packet size */
  53 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  54
  55 #define MRVL_IFACE_NAME_ARG "iface"
  56 #define MRVL_CFG_ARG "cfg"
  57
  58 #define MRVL_BURST_SIZE 64
  59
  60 #define MRVL_ARP_LENGTH 28
  61
  62 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  63
  64 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
  65 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
  66
  67 /** Port Rx offload capabilities */
  68 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
  69                           DEV_RX_OFFLOAD_JUMBO_FRAME | \
  70                           DEV_RX_OFFLOAD_CHECKSUM)
  71
  72 /** Port Tx offloads capabilities */
  73 #define MRVL_TX_OFFLOADS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
  74                           DEV_TX_OFFLOAD_UDP_CKSUM | \
  75                           DEV_TX_OFFLOAD_TCP_CKSUM)
  76
  77 static const char * const valid_args[] = {
  78         MRVL_IFACE_NAME_ARG,
  79         MRVL_CFG_ARG,
  80         NULL
  81 };
  82
  83 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
  84 static struct pp2_hif *hifs[RTE_MAX_LCORE];
  85 static int used_bpools[PP2_NUM_PKT_PROC] = {
  86         [0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED
  87 };
  88
  89 static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
  90 static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
  91 static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
  92
  93 int mrvl_logtype;
  94
  95 struct mrvl_ifnames {
  96         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
  97         int idx;
  98 };
  99
 100 /*
 101  * To use buffer harvesting based on loopback port shadow queue structure
 102  * was introduced for buffers information bookkeeping.
 103  *
 104  * Before sending the packet, related buffer information (pp2_buff_inf) is
 105  * stored in shadow queue. After packet is transmitted no longer used
 106  * packet buffer is released back to it's original hardware pool,
 107  * on condition it originated from interface.
 108  * In case it  was generated by application itself i.e: mbuf->port field is
 109  * 0xff then its released to software mempool.
 110  */
 111 struct mrvl_shadow_txq {
 112         int head;           /* write index - used when sending buffers */
 113         int tail;           /* read index - used when releasing buffers */
 114         u16 size;           /* queue occupied size */
 115         u16 num_to_release; /* number of buffers sent, that can be released */
 116         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 117 };
 118
 119 struct mrvl_rxq {
 120         struct mrvl_priv *priv;
 121         struct rte_mempool *mp;
 122         int queue_id;
 123         int port_id;
 124         int cksum_enabled;
 125         uint64_t bytes_recv;
 126         uint64_t drop_mac;
 127 };
 128
 129 struct mrvl_txq {
 130         struct mrvl_priv *priv;
 131         int queue_id;
 132         int port_id;
 133         uint64_t bytes_sent;
 134         struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE];
 135         int tx_deferred_start;
 136 };
 137
 138 static int mrvl_lcore_first;
 139 static int mrvl_lcore_last;
 140 static int mrvl_dev_num;
 141
 142 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
 143 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
 144                         struct pp2_hif *hif, unsigned int core_id,
 145                         struct mrvl_shadow_txq *sq, int qid, int force);
 146
 147 #define MRVL_XSTATS_TBL_ENTRY(name) { \
 148         #name, offsetof(struct pp2_ppio_statistics, name),      \
 149         sizeof(((struct pp2_ppio_statistics *)0)->name)         \
 150 }
 151
 152 /* Table with xstats data */
 153 static struct {
 154         const char *name;
 155         unsigned int offset;
 156         unsigned int size;
 157 } mrvl_xstats_tbl[] = {
 158         MRVL_XSTATS_TBL_ENTRY(rx_bytes),
 159         MRVL_XSTATS_TBL_ENTRY(rx_packets),
 160         MRVL_XSTATS_TBL_ENTRY(rx_unicast_packets),
 161         MRVL_XSTATS_TBL_ENTRY(rx_errors),
 162         MRVL_XSTATS_TBL_ENTRY(rx_fullq_dropped),
 163         MRVL_XSTATS_TBL_ENTRY(rx_bm_dropped),
 164         MRVL_XSTATS_TBL_ENTRY(rx_early_dropped),
 165         MRVL_XSTATS_TBL_ENTRY(rx_fifo_dropped),
 166         MRVL_XSTATS_TBL_ENTRY(rx_cls_dropped),
 167         MRVL_XSTATS_TBL_ENTRY(tx_bytes),
 168         MRVL_XSTATS_TBL_ENTRY(tx_packets),
 169         MRVL_XSTATS_TBL_ENTRY(tx_unicast_packets),
 170         MRVL_XSTATS_TBL_ENTRY(tx_errors)
 171 };
 172
 173 static inline int
 174 mrvl_get_bpool_size(int pp2_id, int pool_id)
 175 {
 176         int i;
 177         int size = 0;
 178
 179         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 180                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 181
 182         return size;
 183 }
 184
 185 static inline int
 186 mrvl_reserve_bit(int *bitmap, int max)
 187 {
 188         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 189
 190         if (n >= max)
 191                 return -1;
 192
 193         *bitmap |= 1 << n;
 194
 195         return n;
 196 }
 197
 198 static int
 199 mrvl_init_hif(int core_id)
 200 {
 201         struct pp2_hif_params params;
 202         char match[MRVL_MATCH_LEN];
 203         int ret;
 204
 205         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
 206         if (ret < 0) {
 207                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 208                 return ret;
 209         }
 210
 211         snprintf(match, sizeof(match), "hif-%d", ret);
 212         memset(&params, 0, sizeof(params));
 213         params.match = match;
 214         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
 215         ret = pp2_hif_init(&params, &hifs[core_id]);
 216         if (ret) {
 217                 MRVL_LOG(ERR, "Failed to initialize hif %d", core_id);
 218                 return ret;
 219         }
 220
 221         return 0;
 222 }
 223
 224 static inline struct pp2_hif*
 225 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
 226 {
 227         int ret;
 228
 229         if (likely(hifs[core_id] != NULL))
 230                 return hifs[core_id];
 231
 232         rte_spinlock_lock(&priv->lock);
 233
 234         ret = mrvl_init_hif(core_id);
 235         if (ret < 0) {
 236                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 237                 goto out;
 238         }
 239
 240         if (core_id < mrvl_lcore_first)
 241                 mrvl_lcore_first = core_id;
 242
 243         if (core_id > mrvl_lcore_last)
 244                 mrvl_lcore_last = core_id;
 245 out:
 246         rte_spinlock_unlock(&priv->lock);
 247
 248         return hifs[core_id];
 249 }
 250
 251 /**
 252  * Configure rss based on dpdk rss configuration.
 253  *
 254  * @param priv
 255  *   Pointer to private structure.
 256  * @param rss_conf
 257  *   Pointer to RSS configuration.
 258  *
 259  * @return
 260  *   0 on success, negative error value otherwise.
 261  */
 262 static int
 263 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 264 {
 265         if (rss_conf->rss_key)
 266                 MRVL_LOG(WARNING, "Changing hash key is not supported");
 267
 268         if (rss_conf->rss_hf == 0) {
 269                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 270         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 271                 priv->ppio_params.inqs_params.hash_type =
 272                         PP2_PPIO_HASH_T_2_TUPLE;
 273         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 274                 priv->ppio_params.inqs_params.hash_type =
 275                         PP2_PPIO_HASH_T_5_TUPLE;
 276                 priv->rss_hf_tcp = 1;
 277         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 278                 priv->ppio_params.inqs_params.hash_type =
 279                         PP2_PPIO_HASH_T_5_TUPLE;
 280                 priv->rss_hf_tcp = 0;
 281         } else {
 282                 return -EINVAL;
 283         }
 284
 285         return 0;
 286 }
 287
 288 /**
 289  * Ethernet device configuration.
 290  *
 291  * Prepare the driver for a given number of TX and RX queues and
 292  * configure RSS.
 293  *
 294  * @param dev
 295  *   Pointer to Ethernet device structure.
 296  *
 297  * @return
 298  *   0 on success, negative error value otherwise.
 299  */
 300 static int
 301 mrvl_dev_configure(struct rte_eth_dev *dev)
 302 {
 303         struct mrvl_priv *priv = dev->data->dev_private;
 304         int ret;
 305
 306         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 307             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 308                 MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
 309                         dev->data->dev_conf.rxmode.mq_mode);
 310                 return -EINVAL;
 311         }
 312
 313         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 314                 MRVL_LOG(INFO, "Split headers not supported");
 315                 return -EINVAL;
 316         }
 317
 318         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 319                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 320                                  ETHER_HDR_LEN - ETHER_CRC_LEN;
 321
 322         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 323                                   dev->data->nb_rx_queues);
 324         if (ret < 0)
 325                 return ret;
 326
 327         ret = mrvl_configure_txqs(priv, dev->data->port_id,
 328                                   dev->data->nb_tx_queues);
 329         if (ret < 0)
 330                 return ret;
 331
 332         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 333         priv->ppio_params.maintain_stats = 1;
 334         priv->nb_rx_queues = dev->data->nb_rx_queues;
 335
 336         if (dev->data->nb_rx_queues == 1 &&
 337             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 338                 MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
 339                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 340
 341                 return 0;
 342         }
 343
 344         return mrvl_configure_rss(priv,
 345                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
 346 }
 347
 348 /**
 349  * DPDK callback to change the MTU.
 350  *
 351  * Setting the MTU affects hardware MRU (packets larger than the MRU
 352  * will be dropped).
 353  *
 354  * @param dev
 355  *   Pointer to Ethernet device structure.
 356  * @param mtu
 357  *   New MTU.
 358  *
 359  * @return
 360  *   0 on success, negative error value otherwise.
 361  */
 362 static int
 363 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 364 {
 365         struct mrvl_priv *priv = dev->data->dev_private;
 366         /* extra MV_MH_SIZE bytes are required for Marvell tag */
 367         uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
 368         int ret;
 369
 370         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
 371                 return -EINVAL;
 372
 373         if (!priv->ppio)
 374                 return 0;
 375
 376         ret = pp2_ppio_set_mru(priv->ppio, mru);
 377         if (ret)
 378                 return ret;
 379
 380         return pp2_ppio_set_mtu(priv->ppio, mtu);
 381 }
 382
 383 /**
 384  * DPDK callback to bring the link up.
 385  *
 386  * @param dev
 387  *   Pointer to Ethernet device structure.
 388  *
 389  * @return
 390  *   0 on success, negative error value otherwise.
 391  */
 392 static int
 393 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 394 {
 395         struct mrvl_priv *priv = dev->data->dev_private;
 396         int ret;
 397
 398         if (!priv->ppio)
 399                 return -EPERM;
 400
 401         ret = pp2_ppio_enable(priv->ppio);
 402         if (ret)
 403                 return ret;
 404
 405         /*
 406          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 407          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 408          * PP2_TRAFFIC_INGRESS_EGRESS.
 409          *
 410          * Set mtu to default DPDK value here.
 411          */
 412         ret = mrvl_mtu_set(dev, dev->data->mtu);
 413         if (ret)
 414                 pp2_ppio_disable(priv->ppio);
 415
 416         return ret;
 417 }
 418
 419 /**
 420  * DPDK callback to bring the link down.
 421  *
 422  * @param dev
 423  *   Pointer to Ethernet device structure.
 424  *
 425  * @return
 426  *   0 on success, negative error value otherwise.
 427  */
 428 static int
 429 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 430 {
 431         struct mrvl_priv *priv = dev->data->dev_private;
 432
 433         if (!priv->ppio)
 434                 return -EPERM;
 435
 436         return pp2_ppio_disable(priv->ppio);
 437 }
 438
 439 /**
 440  * DPDK callback to start tx queue.
 441  *
 442  * @param dev
 443  *   Pointer to Ethernet device structure.
 444  * @param queue_id
 445  *   Transmit queue index.
 446  *
 447  * @return
 448  *   0 on success, negative error value otherwise.
 449  */
 450 static int
 451 mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
 452 {
 453         struct mrvl_priv *priv = dev->data->dev_private;
 454         int ret;
 455
 456         if (!priv)
 457                 return -EPERM;
 458
 459         /* passing 1 enables given tx queue */
 460         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1);
 461         if (ret) {
 462                 MRVL_LOG(ERR, "Failed to start txq %d", queue_id);
 463                 return ret;
 464         }
 465
 466         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 467
 468         return 0;
 469 }
 470
 471 /**
 472  * DPDK callback to stop tx queue.
 473  *
 474  * @param dev
 475  *   Pointer to Ethernet device structure.
 476  * @param queue_id
 477  *   Transmit queue index.
 478  *
 479  * @return
 480  *   0 on success, negative error value otherwise.
 481  */
 482 static int
 483 mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
 484 {
 485         struct mrvl_priv *priv = dev->data->dev_private;
 486         int ret;
 487
 488         if (!priv->ppio)
 489                 return -EPERM;
 490
 491         /* passing 0 disables given tx queue */
 492         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0);
 493         if (ret) {
 494                 MRVL_LOG(ERR, "Failed to stop txq %d", queue_id);
 495                 return ret;
 496         }
 497
 498         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 499
 500         return 0;
 501 }
 502
 503 /**
 504  * DPDK callback to start the device.
 505  *
 506  * @param dev
 507  *   Pointer to Ethernet device structure.
 508  *
 509  * @return
 510  *   0 on success, negative errno value on failure.
 511  */
 512 static int
 513 mrvl_dev_start(struct rte_eth_dev *dev)
 514 {
 515         struct mrvl_priv *priv = dev->data->dev_private;
 516         char match[MRVL_MATCH_LEN];
 517         int ret = 0, i, def_init_size;
 518
 519         snprintf(match, sizeof(match), "ppio-%d:%d",
 520                  priv->pp_id, priv->ppio_id);
 521         priv->ppio_params.match = match;
 522
 523         /*
 524          * Calculate the minimum bpool size for refill feature as follows:
 525          * 2 default burst sizes multiply by number of rx queues.
 526          * If the bpool size will be below this value, new buffers will
 527          * be added to the pool.
 528          */
 529         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 530
 531         /* In case initial bpool size configured in queues setup is
 532          * smaller than minimum size add more buffers
 533          */
 534         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
 535         if (priv->bpool_init_size < def_init_size) {
 536                 int buffs_to_add = def_init_size - priv->bpool_init_size;
 537
 538                 priv->bpool_init_size += buffs_to_add;
 539                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
 540                 if (ret)
 541                         MRVL_LOG(ERR, "Failed to add buffers to bpool");
 542         }
 543
 544         /*
 545          * Calculate the maximum bpool size for refill feature as follows:
 546          * maximum number of descriptors in rx queue multiply by number
 547          * of rx queues plus minimum bpool size.
 548          * In case the bpool size will exceed this value, superfluous buffers
 549          * will be removed
 550          */
 551         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
 552                                 priv->bpool_min_size;
 553
 554         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 555         if (ret) {
 556                 MRVL_LOG(ERR, "Failed to init ppio");
 557                 return ret;
 558         }
 559
 560         /*
 561          * In case there are some some stale uc/mc mac addresses flush them
 562          * here. It cannot be done during mrvl_dev_close() as port information
 563          * is already gone at that point (due to pp2_ppio_deinit() in
 564          * mrvl_dev_stop()).
 565          */
 566         if (!priv->uc_mc_flushed) {
 567                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 568                 if (ret) {
 569                         MRVL_LOG(ERR,
 570                                 "Failed to flush uc/mc filter list");
 571                         goto out;
 572                 }
 573                 priv->uc_mc_flushed = 1;
 574         }
 575
 576         if (!priv->vlan_flushed) {
 577                 ret = pp2_ppio_flush_vlan(priv->ppio);
 578                 if (ret) {
 579                         MRVL_LOG(ERR, "Failed to flush vlan list");
 580                         /*
 581                          * TODO
 582                          * once pp2_ppio_flush_vlan() is supported jump to out
 583                          * goto out;
 584                          */
 585                 }
 586                 priv->vlan_flushed = 1;
 587         }
 588
 589         /* For default QoS config, don't start classifier. */
 590         if (mrvl_qos_cfg) {
 591                 ret = mrvl_start_qos_mapping(priv);
 592                 if (ret) {
 593                         MRVL_LOG(ERR, "Failed to setup QoS mapping");
 594                         goto out;
 595                 }
 596         }
 597
 598         ret = mrvl_dev_set_link_up(dev);
 599         if (ret) {
 600                 MRVL_LOG(ERR, "Failed to set link up");
 601                 goto out;
 602         }
 603
 604         /* start tx queues */
 605         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 606                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 607
 608                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
 609
 610                 if (!txq->tx_deferred_start)
 611                         continue;
 612
 613                 /*
 614                  * All txqs are started by default. Stop them
 615                  * so that tx_deferred_start works as expected.
 616                  */
 617                 ret = mrvl_tx_queue_stop(dev, i);
 618                 if (ret)
 619                         goto out;
 620         }
 621
 622         return 0;
 623 out:
 624         MRVL_LOG(ERR, "Failed to start device");
 625         pp2_ppio_deinit(priv->ppio);
 626         return ret;
 627 }
 628
 629 /**
 630  * Flush receive queues.
 631  *
 632  * @param dev
 633  *   Pointer to Ethernet device structure.
 634  */
 635 static void
 636 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 637 {
 638         int i;
 639
 640         MRVL_LOG(INFO, "Flushing rx queues");
 641         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 642                 int ret, num;
 643
 644                 do {
 645                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 646                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 647
 648                         num = MRVL_PP2_RXD_MAX;
 649                         ret = pp2_ppio_recv(q->priv->ppio,
 650                                             q->priv->rxq_map[q->queue_id].tc,
 651                                             q->priv->rxq_map[q->queue_id].inq,
 652                                             descs, (uint16_t *)&num);
 653                 } while (ret == 0 && num);
 654         }
 655 }
 656
 657 /**
 658  * Flush transmit shadow queues.
 659  *
 660  * @param dev
 661  *   Pointer to Ethernet device structure.
 662  */
 663 static void
 664 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 665 {
 666         int i, j;
 667         struct mrvl_txq *txq;
 668
 669         MRVL_LOG(INFO, "Flushing tx shadow queues");
 670         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 671                 txq = (struct mrvl_txq *)dev->data->tx_queues[i];
 672
 673                 for (j = 0; j < RTE_MAX_LCORE; j++) {
 674                         struct mrvl_shadow_txq *sq;
 675
 676                         if (!hifs[j])
 677                                 continue;
 678
 679                         sq = &txq->shadow_txqs[j];
 680                         mrvl_free_sent_buffers(txq->priv->ppio,
 681                                 hifs[j], j, sq, txq->queue_id, 1);
 682                         while (sq->tail != sq->head) {
 683                                 uint64_t addr = cookie_addr_high |
 684                                         sq->ent[sq->tail].buff.cookie;
 685                                 rte_pktmbuf_free(
 686                                         (struct rte_mbuf *)addr);
 687                                 sq->tail = (sq->tail + 1) &
 688                                             MRVL_PP2_TX_SHADOWQ_MASK;
 689                         }
 690                         memset(sq, 0, sizeof(*sq));
 691                 }
 692         }
 693 }
 694
 695 /**
 696  * Flush hardware bpool (buffer-pool).
 697  *
 698  * @param dev
 699  *   Pointer to Ethernet device structure.
 700  */
 701 static void
 702 mrvl_flush_bpool(struct rte_eth_dev *dev)
 703 {
 704         struct mrvl_priv *priv = dev->data->dev_private;
 705         struct pp2_hif *hif;
 706         uint32_t num;
 707         int ret;
 708         unsigned int core_id = rte_lcore_id();
 709
 710         if (core_id == LCORE_ID_ANY)
 711                 core_id = 0;
 712
 713         hif = mrvl_get_hif(priv, core_id);
 714
 715         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 716         if (ret) {
 717                 MRVL_LOG(ERR, "Failed to get bpool buffers number");
 718                 return;
 719         }
 720
 721         while (num--) {
 722                 struct pp2_buff_inf inf;
 723                 uint64_t addr;
 724
 725                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
 726                 if (ret)
 727                         break;
 728
 729                 addr = cookie_addr_high | inf.cookie;
 730                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 731         }
 732 }
 733
 734 /**
 735  * DPDK callback to stop the device.
 736  *
 737  * @param dev
 738  *   Pointer to Ethernet device structure.
 739  */
 740 static void
 741 mrvl_dev_stop(struct rte_eth_dev *dev)
 742 {
 743         struct mrvl_priv *priv = dev->data->dev_private;
 744
 745         mrvl_dev_set_link_down(dev);
 746         mrvl_flush_rx_queues(dev);
 747         mrvl_flush_tx_shadow_queues(dev);
 748         if (priv->cls_tbl) {
 749                 pp2_cls_tbl_deinit(priv->cls_tbl);
 750                 priv->cls_tbl = NULL;
 751         }
 752         if (priv->qos_tbl) {
 753                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 754                 priv->qos_tbl = NULL;
 755         }
 756         if (priv->ppio)
 757                 pp2_ppio_deinit(priv->ppio);
 758         priv->ppio = NULL;
 759
 760         /* policer must be released after ppio deinitialization */
 761         if (priv->policer) {
 762                 pp2_cls_plcr_deinit(priv->policer);
 763                 priv->policer = NULL;
 764         }
 765 }
 766
 767 /**
 768  * DPDK callback to close the device.
 769  *
 770  * @param dev
 771  *   Pointer to Ethernet device structure.
 772  */
 773 static void
 774 mrvl_dev_close(struct rte_eth_dev *dev)
 775 {
 776         struct mrvl_priv *priv = dev->data->dev_private;
 777         size_t i;
 778
 779         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 780                 struct pp2_ppio_tc_params *tc_params =
 781                         &priv->ppio_params.inqs_params.tcs_params[i];
 782
 783                 if (tc_params->inqs_params) {
 784                         rte_free(tc_params->inqs_params);
 785                         tc_params->inqs_params = NULL;
 786                 }
 787         }
 788
 789         mrvl_flush_bpool(dev);
 790 }
 791
 792 /**
 793  * DPDK callback to retrieve physical link information.
 794  *
 795  * @param dev
 796  *   Pointer to Ethernet device structure.
 797  * @param wait_to_complete
 798  *   Wait for request completion (ignored).
 799  *
 800  * @return
 801  *   0 on success, negative error value otherwise.
 802  */
 803 static int
 804 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 805 {
 806         /*
 807          * TODO
 808          * once MUSDK provides necessary API use it here
 809          */
 810         struct mrvl_priv *priv = dev->data->dev_private;
 811         struct ethtool_cmd edata;
 812         struct ifreq req;
 813         int ret, fd, link_up;
 814
 815         if (!priv->ppio)
 816                 return -EPERM;
 817
 818         edata.cmd = ETHTOOL_GSET;
 819
 820         strcpy(req.ifr_name, dev->data->name);
 821         req.ifr_data = (void *)&edata;
 822
 823         fd = socket(AF_INET, SOCK_DGRAM, 0);
 824         if (fd == -1)
 825                 return -EFAULT;
 826
 827         ret = ioctl(fd, SIOCETHTOOL, &req);
 828         if (ret == -1) {
 829                 close(fd);
 830                 return -EFAULT;
 831         }
 832
 833         close(fd);
 834
 835         switch (ethtool_cmd_speed(&edata)) {
 836         case SPEED_10:
 837                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 838                 break;
 839         case SPEED_100:
 840                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 841                 break;
 842         case SPEED_1000:
 843                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 844                 break;
 845         case SPEED_10000:
 846                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 847                 break;
 848         default:
 849                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 850         }
 851
 852         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 853                                                          ETH_LINK_HALF_DUPLEX;
 854         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 855                                                            ETH_LINK_FIXED;
 856         pp2_ppio_get_link_state(priv->ppio, &link_up);
 857         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
 858
 859         return 0;
 860 }
 861
 862 /**
 863  * DPDK callback to enable promiscuous mode.
 864  *
 865  * @param dev
 866  *   Pointer to Ethernet device structure.
 867  */
 868 static void
 869 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
 870 {
 871         struct mrvl_priv *priv = dev->data->dev_private;
 872         int ret;
 873
 874         if (!priv->ppio)
 875                 return;
 876
 877         if (priv->isolated)
 878                 return;
 879
 880         ret = pp2_ppio_set_promisc(priv->ppio, 1);
 881         if (ret)
 882                 MRVL_LOG(ERR, "Failed to enable promiscuous mode");
 883 }
 884
 885 /**
 886  * DPDK callback to enable allmulti mode.
 887  *
 888  * @param dev
 889  *   Pointer to Ethernet device structure.
 890  */
 891 static void
 892 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
 893 {
 894         struct mrvl_priv *priv = dev->data->dev_private;
 895         int ret;
 896
 897         if (!priv->ppio)
 898                 return;
 899
 900         if (priv->isolated)
 901                 return;
 902
 903         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
 904         if (ret)
 905                 MRVL_LOG(ERR, "Failed enable all-multicast mode");
 906 }
 907
 908 /**
 909  * DPDK callback to disable promiscuous mode.
 910  *
 911  * @param dev
 912  *   Pointer to Ethernet device structure.
 913  */
 914 static void
 915 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
 916 {
 917         struct mrvl_priv *priv = dev->data->dev_private;
 918         int ret;
 919
 920         if (!priv->ppio)
 921                 return;
 922
 923         ret = pp2_ppio_set_promisc(priv->ppio, 0);
 924         if (ret)
 925                 MRVL_LOG(ERR, "Failed to disable promiscuous mode");
 926 }
 927
 928 /**
 929  * DPDK callback to disable allmulticast mode.
 930  *
 931  * @param dev
 932  *   Pointer to Ethernet device structure.
 933  */
 934 static void
 935 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
 936 {
 937         struct mrvl_priv *priv = dev->data->dev_private;
 938         int ret;
 939
 940         if (!priv->ppio)
 941                 return;
 942
 943         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
 944         if (ret)
 945                 MRVL_LOG(ERR, "Failed to disable all-multicast mode");
 946 }
 947
 948 /**
 949  * DPDK callback to remove a MAC address.
 950  *
 951  * @param dev
 952  *   Pointer to Ethernet device structure.
 953  * @param index
 954  *   MAC address index.
 955  */
 956 static void
 957 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 958 {
 959         struct mrvl_priv *priv = dev->data->dev_private;
 960         char buf[ETHER_ADDR_FMT_SIZE];
 961         int ret;
 962
 963         if (!priv->ppio)
 964                 return;
 965
 966         if (priv->isolated)
 967                 return;
 968
 969         ret = pp2_ppio_remove_mac_addr(priv->ppio,
 970                                        dev->data->mac_addrs[index].addr_bytes);
 971         if (ret) {
 972                 ether_format_addr(buf, sizeof(buf),
 973                                   &dev->data->mac_addrs[index]);
 974                 MRVL_LOG(ERR, "Failed to remove mac %s", buf);
 975         }
 976 }
 977
 978 /**
 979  * DPDK callback to add a MAC address.
 980  *
 981  * @param dev
 982  *   Pointer to Ethernet device structure.
 983  * @param mac_addr
 984  *   MAC address to register.
 985  * @param index
 986  *   MAC address index.
 987  * @param vmdq
 988  *   VMDq pool index to associate address with (unused).
 989  *
 990  * @return
 991  *   0 on success, negative error value otherwise.
 992  */
 993 static int
 994 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 995                   uint32_t index, uint32_t vmdq __rte_unused)
 996 {
 997         struct mrvl_priv *priv = dev->data->dev_private;
 998         char buf[ETHER_ADDR_FMT_SIZE];
 999         int ret;
1000
1001         if (priv->isolated)
1002                 return -ENOTSUP;
1003
1004         if (index == 0)
1005                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
1006                 return -1;
1007
1008         if (!priv->ppio)
1009                 return 0;
1010
1011         /*
1012          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
1013          * parameter uc_filter_max. Maximum number of mc addresses is then
1014          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
1015          * 21 respectively.
1016          *
1017          * If more than uc_filter_max uc addresses were added to filter list
1018          * then NIC will switch to promiscuous mode automatically.
1019          *
1020          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
1021          * were added to filter list then NIC will switch to all-multicast mode
1022          * automatically.
1023          */
1024         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
1025         if (ret) {
1026                 ether_format_addr(buf, sizeof(buf), mac_addr);
1027                 MRVL_LOG(ERR, "Failed to add mac %s", buf);
1028                 return -1;
1029         }
1030
1031         return 0;
1032 }
1033
1034 /**
1035  * DPDK callback to set the primary MAC address.
1036  *
1037  * @param dev
1038  *   Pointer to Ethernet device structure.
1039  * @param mac_addr
1040  *   MAC address to register.
1041  *
1042  * @return
1043  *   0 on success, negative error value otherwise.
1044  */
1045 static int
1046 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1047 {
1048         struct mrvl_priv *priv = dev->data->dev_private;
1049         int ret;
1050
1051         if (!priv->ppio)
1052                 return 0;
1053
1054         if (priv->isolated)
1055                 return -ENOTSUP;
1056
1057         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
1058         if (ret) {
1059                 char buf[ETHER_ADDR_FMT_SIZE];
1060                 ether_format_addr(buf, sizeof(buf), mac_addr);
1061                 MRVL_LOG(ERR, "Failed to set mac to %s", buf);
1062         }
1063
1064         return ret;
1065 }
1066
1067 /**
1068  * DPDK callback to get device statistics.
1069  *
1070  * @param dev
1071  *   Pointer to Ethernet device structure.
1072  * @param stats
1073  *   Stats structure output buffer.
1074  *
1075  * @return
1076  *   0 on success, negative error value otherwise.
1077  */
1078 static int
1079 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1080 {
1081         struct mrvl_priv *priv = dev->data->dev_private;
1082         struct pp2_ppio_statistics ppio_stats;
1083         uint64_t drop_mac = 0;
1084         unsigned int i, idx, ret;
1085
1086         if (!priv->ppio)
1087                 return -EPERM;
1088
1089         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1090                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1091                 struct pp2_ppio_inq_statistics rx_stats;
1092
1093                 if (!rxq)
1094                         continue;
1095
1096                 idx = rxq->queue_id;
1097                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1098                         MRVL_LOG(ERR,
1099                                 "rx queue %d stats out of range (0 - %d)",
1100                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1101                         continue;
1102                 }
1103
1104                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
1105                                                   priv->rxq_map[idx].tc,
1106                                                   priv->rxq_map[idx].inq,
1107                                                   &rx_stats, 0);
1108                 if (unlikely(ret)) {
1109                         MRVL_LOG(ERR,
1110                                 "Failed to update rx queue %d stats", idx);
1111                         break;
1112                 }
1113
1114                 stats->q_ibytes[idx] = rxq->bytes_recv;
1115                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
1116                 stats->q_errors[idx] = rx_stats.drop_early +
1117                                        rx_stats.drop_fullq +
1118                                        rx_stats.drop_bm +
1119                                        rxq->drop_mac;
1120                 stats->ibytes += rxq->bytes_recv;
1121                 drop_mac += rxq->drop_mac;
1122         }
1123
1124         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1125                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1126                 struct pp2_ppio_outq_statistics tx_stats;
1127
1128                 if (!txq)
1129                         continue;
1130
1131                 idx = txq->queue_id;
1132                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1133                         MRVL_LOG(ERR,
1134                                 "tx queue %d stats out of range (0 - %d)",
1135                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1136                 }
1137
1138                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1139                                                    &tx_stats, 0);
1140                 if (unlikely(ret)) {
1141                         MRVL_LOG(ERR,
1142                                 "Failed to update tx queue %d stats", idx);
1143                         break;
1144                 }
1145
1146                 stats->q_opackets[idx] = tx_stats.deq_desc;
1147                 stats->q_obytes[idx] = txq->bytes_sent;
1148                 stats->obytes += txq->bytes_sent;
1149         }
1150
1151         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1152         if (unlikely(ret)) {
1153                 MRVL_LOG(ERR, "Failed to update port statistics");
1154                 return ret;
1155         }
1156
1157         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1158         stats->opackets += ppio_stats.tx_packets;
1159         stats->imissed += ppio_stats.rx_fullq_dropped +
1160                           ppio_stats.rx_bm_dropped +
1161                           ppio_stats.rx_early_dropped +
1162                           ppio_stats.rx_fifo_dropped +
1163                           ppio_stats.rx_cls_dropped;
1164         stats->ierrors = drop_mac;
1165
1166         return 0;
1167 }
1168
1169 /**
1170  * DPDK callback to clear device statistics.
1171  *
1172  * @param dev
1173  *   Pointer to Ethernet device structure.
1174  */
1175 static void
1176 mrvl_stats_reset(struct rte_eth_dev *dev)
1177 {
1178         struct mrvl_priv *priv = dev->data->dev_private;
1179         int i;
1180
1181         if (!priv->ppio)
1182                 return;
1183
1184         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1185                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1186
1187                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1188                                             priv->rxq_map[i].inq, NULL, 1);
1189                 rxq->bytes_recv = 0;
1190                 rxq->drop_mac = 0;
1191         }
1192
1193         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1194                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1195
1196                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1197                 txq->bytes_sent = 0;
1198         }
1199
1200         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1201 }
1202
1203 /**
1204  * DPDK callback to get extended statistics.
1205  *
1206  * @param dev
1207  *   Pointer to Ethernet device structure.
1208  * @param stats
1209  *   Pointer to xstats table.
1210  * @param n
1211  *   Number of entries in xstats table.
1212  * @return
1213  *   Negative value on error, number of read xstats otherwise.
1214  */
1215 static int
1216 mrvl_xstats_get(struct rte_eth_dev *dev,
1217                 struct rte_eth_xstat *stats, unsigned int n)
1218 {
1219         struct mrvl_priv *priv = dev->data->dev_private;
1220         struct pp2_ppio_statistics ppio_stats;
1221         unsigned int i;
1222
1223         if (!stats)
1224                 return 0;
1225
1226         pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1227         for (i = 0; i < n && i < RTE_DIM(mrvl_xstats_tbl); i++) {
1228                 uint64_t val;
1229
1230                 if (mrvl_xstats_tbl[i].size == sizeof(uint32_t))
1231                         val = *(uint32_t *)((uint8_t *)&ppio_stats +
1232                                             mrvl_xstats_tbl[i].offset);
1233                 else if (mrvl_xstats_tbl[i].size == sizeof(uint64_t))
1234                         val = *(uint64_t *)((uint8_t *)&ppio_stats +
1235                                             mrvl_xstats_tbl[i].offset);
1236                 else
1237                         return -EINVAL;
1238
1239                 stats[i].id = i;
1240                 stats[i].value = val;
1241         }
1242
1243         return n;
1244 }
1245
1246 /**
1247  * DPDK callback to reset extended statistics.
1248  *
1249  * @param dev
1250  *   Pointer to Ethernet device structure.
1251  */
1252 static void
1253 mrvl_xstats_reset(struct rte_eth_dev *dev)
1254 {
1255         mrvl_stats_reset(dev);
1256 }
1257
1258 /**
1259  * DPDK callback to get extended statistics names.
1260  *
1261  * @param dev (unused)
1262  *   Pointer to Ethernet device structure.
1263  * @param xstats_names
1264  *   Pointer to xstats names table.
1265  * @param size
1266  *   Size of the xstats names table.
1267  * @return
1268  *   Number of read names.
1269  */
1270 static int
1271 mrvl_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
1272                       struct rte_eth_xstat_name *xstats_names,
1273                       unsigned int size)
1274 {
1275         unsigned int i;
1276
1277         if (!xstats_names)
1278                 return RTE_DIM(mrvl_xstats_tbl);
1279
1280         for (i = 0; i < size && i < RTE_DIM(mrvl_xstats_tbl); i++)
1281                 snprintf(xstats_names[i].name, RTE_ETH_XSTATS_NAME_SIZE, "%s",
1282                          mrvl_xstats_tbl[i].name);
1283
1284         return size;
1285 }
1286
1287 /**
1288  * DPDK callback to get information about the device.
1289  *
1290  * @param dev
1291  *   Pointer to Ethernet device structure (unused).
1292  * @param info
1293  *   Info structure output buffer.
1294  */
1295 static void
1296 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
1297                    struct rte_eth_dev_info *info)
1298 {
1299         info->speed_capa = ETH_LINK_SPEED_10M |
1300                            ETH_LINK_SPEED_100M |
1301                            ETH_LINK_SPEED_1G |
1302                            ETH_LINK_SPEED_10G;
1303
1304         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1305         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1306         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1307
1308         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1309         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1310         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1311
1312         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1313         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1314         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1315
1316         info->rx_offload_capa = MRVL_RX_OFFLOADS;
1317         info->rx_queue_offload_capa = MRVL_RX_OFFLOADS;
1318
1319         info->tx_offload_capa = MRVL_TX_OFFLOADS;
1320         info->tx_queue_offload_capa = MRVL_TX_OFFLOADS;
1321
1322         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1323                                        ETH_RSS_NONFRAG_IPV4_TCP |
1324                                        ETH_RSS_NONFRAG_IPV4_UDP;
1325
1326         /* By default packets are dropped if no descriptors are available */
1327         info->default_rxconf.rx_drop_en = 1;
1328
1329         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1330 }
1331
1332 /**
1333  * Return supported packet types.
1334  *
1335  * @param dev
1336  *   Pointer to Ethernet device structure (unused).
1337  *
1338  * @return
1339  *   Const pointer to the table with supported packet types.
1340  */
1341 static const uint32_t *
1342 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1343 {
1344         static const uint32_t ptypes[] = {
1345                 RTE_PTYPE_L2_ETHER,
1346                 RTE_PTYPE_L2_ETHER_VLAN,
1347                 RTE_PTYPE_L2_ETHER_QINQ,
1348                 RTE_PTYPE_L3_IPV4,
1349                 RTE_PTYPE_L3_IPV4_EXT,
1350                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1351                 RTE_PTYPE_L3_IPV6,
1352                 RTE_PTYPE_L3_IPV6_EXT,
1353                 RTE_PTYPE_L2_ETHER_ARP,
1354                 RTE_PTYPE_L4_TCP,
1355                 RTE_PTYPE_L4_UDP
1356         };
1357
1358         return ptypes;
1359 }
1360
1361 /**
1362  * DPDK callback to get information about specific receive queue.
1363  *
1364  * @param dev
1365  *   Pointer to Ethernet device structure.
1366  * @param rx_queue_id
1367  *   Receive queue index.
1368  * @param qinfo
1369  *   Receive queue information structure.
1370  */
1371 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1372                               struct rte_eth_rxq_info *qinfo)
1373 {
1374         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1375         struct mrvl_priv *priv = dev->data->dev_private;
1376         int inq = priv->rxq_map[rx_queue_id].inq;
1377         int tc = priv->rxq_map[rx_queue_id].tc;
1378         struct pp2_ppio_tc_params *tc_params =
1379                 &priv->ppio_params.inqs_params.tcs_params[tc];
1380
1381         qinfo->mp = q->mp;
1382         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1383 }
1384
1385 /**
1386  * DPDK callback to get information about specific transmit queue.
1387  *
1388  * @param dev
1389  *   Pointer to Ethernet device structure.
1390  * @param tx_queue_id
1391  *   Transmit queue index.
1392  * @param qinfo
1393  *   Transmit queue information structure.
1394  */
1395 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1396                               struct rte_eth_txq_info *qinfo)
1397 {
1398         struct mrvl_priv *priv = dev->data->dev_private;
1399         struct mrvl_txq *txq = dev->data->tx_queues[tx_queue_id];
1400
1401         qinfo->nb_desc =
1402                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1403         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1404 }
1405
1406 /**
1407  * DPDK callback to Configure a VLAN filter.
1408  *
1409  * @param dev
1410  *   Pointer to Ethernet device structure.
1411  * @param vlan_id
1412  *   VLAN ID to filter.
1413  * @param on
1414  *   Toggle filter.
1415  *
1416  * @return
1417  *   0 on success, negative error value otherwise.
1418  */
1419 static int
1420 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1421 {
1422         struct mrvl_priv *priv = dev->data->dev_private;
1423
1424         if (!priv->ppio)
1425                 return -EPERM;
1426
1427         if (priv->isolated)
1428                 return -ENOTSUP;
1429
1430         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1431                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1432 }
1433
1434 /**
1435  * Release buffers to hardware bpool (buffer-pool)
1436  *
1437  * @param rxq
1438  *   Receive queue pointer.
1439  * @param num
1440  *   Number of buffers to release to bpool.
1441  *
1442  * @return
1443  *   0 on success, negative error value otherwise.
1444  */
1445 static int
1446 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1447 {
1448         struct buff_release_entry entries[MRVL_PP2_RXD_MAX];
1449         struct rte_mbuf *mbufs[MRVL_PP2_RXD_MAX];
1450         int i, ret;
1451         unsigned int core_id;
1452         struct pp2_hif *hif;
1453         struct pp2_bpool *bpool;
1454
1455         core_id = rte_lcore_id();
1456         if (core_id == LCORE_ID_ANY)
1457                 core_id = 0;
1458
1459         hif = mrvl_get_hif(rxq->priv, core_id);
1460         if (!hif)
1461                 return -1;
1462
1463         bpool = rxq->priv->bpool;
1464
1465         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1466         if (ret)
1467                 return ret;
1468
1469         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1470                 cookie_addr_high =
1471                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1472
1473         for (i = 0; i < num; i++) {
1474                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1475                         != cookie_addr_high) {
1476                         MRVL_LOG(ERR,
1477                                 "mbuf virtual addr high 0x%lx out of range",
1478                                 (uint64_t)mbufs[i] >> 32);
1479                         goto out;
1480                 }
1481
1482                 entries[i].buff.addr =
1483                         rte_mbuf_data_iova_default(mbufs[i]);
1484                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
1485                 entries[i].bpool = bpool;
1486         }
1487
1488         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1489         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1490
1491         if (i != num)
1492                 goto out;
1493
1494         return 0;
1495 out:
1496         for (; i < num; i++)
1497                 rte_pktmbuf_free(mbufs[i]);
1498
1499         return -1;
1500 }
1501
1502 /**
1503  * DPDK callback to configure the receive queue.
1504  *
1505  * @param dev
1506  *   Pointer to Ethernet device structure.
1507  * @param idx
1508  *   RX queue index.
1509  * @param desc
1510  *   Number of descriptors to configure in queue.
1511  * @param socket
1512  *   NUMA socket on which memory must be allocated.
1513  * @param conf
1514  *   Thresholds parameters.
1515  * @param mp
1516  *   Memory pool for buffer allocations.
1517  *
1518  * @return
1519  *   0 on success, negative error value otherwise.
1520  */
1521 static int
1522 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1523                     unsigned int socket,
1524                     const struct rte_eth_rxconf *conf,
1525                     struct rte_mempool *mp)
1526 {
1527         struct mrvl_priv *priv = dev->data->dev_private;
1528         struct mrvl_rxq *rxq;
1529         uint32_t min_size,
1530                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1531         int ret, tc, inq;
1532         uint64_t offloads;
1533
1534         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1535
1536         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1537                 /*
1538                  * Unknown TC mapping, mapping will not have a correct queue.
1539                  */
1540                 MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu",
1541                         idx, priv->ppio_id);
1542                 return -EFAULT;
1543         }
1544
1545         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
1546                    MRVL_PKT_EFFEC_OFFS;
1547         if (min_size < max_rx_pkt_len) {
1548                 MRVL_LOG(ERR,
1549                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.",
1550                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
1551                         MRVL_PKT_EFFEC_OFFS,
1552                         max_rx_pkt_len);
1553                 return -EINVAL;
1554         }
1555
1556         if (dev->data->rx_queues[idx]) {
1557                 rte_free(dev->data->rx_queues[idx]);
1558                 dev->data->rx_queues[idx] = NULL;
1559         }
1560
1561         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1562         if (!rxq)
1563                 return -ENOMEM;
1564
1565         rxq->priv = priv;
1566         rxq->mp = mp;
1567         rxq->cksum_enabled = offloads & DEV_RX_OFFLOAD_IPV4_CKSUM;
1568         rxq->queue_id = idx;
1569         rxq->port_id = dev->data->port_id;
1570         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1571
1572         tc = priv->rxq_map[rxq->queue_id].tc,
1573         inq = priv->rxq_map[rxq->queue_id].inq;
1574         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1575                 desc;
1576
1577         ret = mrvl_fill_bpool(rxq, desc);
1578         if (ret) {
1579                 rte_free(rxq);
1580                 return ret;
1581         }
1582
1583         priv->bpool_init_size += desc;
1584
1585         dev->data->rx_queues[idx] = rxq;
1586
1587         return 0;
1588 }
1589
1590 /**
1591  * DPDK callback to release the receive queue.
1592  *
1593  * @param rxq
1594  *   Generic receive queue pointer.
1595  */
1596 static void
1597 mrvl_rx_queue_release(void *rxq)
1598 {
1599         struct mrvl_rxq *q = rxq;
1600         struct pp2_ppio_tc_params *tc_params;
1601         int i, num, tc, inq;
1602         struct pp2_hif *hif;
1603         unsigned int core_id = rte_lcore_id();
1604
1605         if (core_id == LCORE_ID_ANY)
1606                 core_id = 0;
1607
1608         if (!q)
1609                 return;
1610
1611         hif = mrvl_get_hif(q->priv, core_id);
1612
1613         if (!hif)
1614                 return;
1615
1616         tc = q->priv->rxq_map[q->queue_id].tc;
1617         inq = q->priv->rxq_map[q->queue_id].inq;
1618         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1619         num = tc_params->inqs_params[inq].size;
1620         for (i = 0; i < num; i++) {
1621                 struct pp2_buff_inf inf;
1622                 uint64_t addr;
1623
1624                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
1625                 addr = cookie_addr_high | inf.cookie;
1626                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1627         }
1628
1629         rte_free(q);
1630 }
1631
1632 /**
1633  * DPDK callback to configure the transmit queue.
1634  *
1635  * @param dev
1636  *   Pointer to Ethernet device structure.
1637  * @param idx
1638  *   Transmit queue index.
1639  * @param desc
1640  *   Number of descriptors to configure in the queue.
1641  * @param socket
1642  *   NUMA socket on which memory must be allocated.
1643  * @param conf
1644  *   Tx queue configuration parameters.
1645  *
1646  * @return
1647  *   0 on success, negative error value otherwise.
1648  */
1649 static int
1650 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1651                     unsigned int socket,
1652                     const struct rte_eth_txconf *conf)
1653 {
1654         struct mrvl_priv *priv = dev->data->dev_private;
1655         struct mrvl_txq *txq;
1656
1657         if (dev->data->tx_queues[idx]) {
1658                 rte_free(dev->data->tx_queues[idx]);
1659                 dev->data->tx_queues[idx] = NULL;
1660         }
1661
1662         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1663         if (!txq)
1664                 return -ENOMEM;
1665
1666         txq->priv = priv;
1667         txq->queue_id = idx;
1668         txq->port_id = dev->data->port_id;
1669         txq->tx_deferred_start = conf->tx_deferred_start;
1670         dev->data->tx_queues[idx] = txq;
1671
1672         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1673
1674         return 0;
1675 }
1676
1677 /**
1678  * DPDK callback to release the transmit queue.
1679  *
1680  * @param txq
1681  *   Generic transmit queue pointer.
1682  */
1683 static void
1684 mrvl_tx_queue_release(void *txq)
1685 {
1686         struct mrvl_txq *q = txq;
1687
1688         if (!q)
1689                 return;
1690
1691         rte_free(q);
1692 }
1693
1694 /**
1695  * DPDK callback to get flow control configuration.
1696  *
1697  * @param dev
1698  *  Pointer to Ethernet device structure.
1699  * @param fc_conf
1700  *  Pointer to the flow control configuration.
1701  *
1702  * @return
1703  *  0 on success, negative error value otherwise.
1704  */
1705 static int
1706 mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1707 {
1708         struct mrvl_priv *priv = dev->data->dev_private;
1709         int ret, en;
1710
1711         if (!priv)
1712                 return -EPERM;
1713
1714         ret = pp2_ppio_get_rx_pause(priv->ppio, &en);
1715         if (ret) {
1716                 MRVL_LOG(ERR, "Failed to read rx pause state");
1717                 return ret;
1718         }
1719
1720         fc_conf->mode = en ? RTE_FC_RX_PAUSE : RTE_FC_NONE;
1721
1722         return 0;
1723 }
1724
1725 /**
1726  * DPDK callback to set flow control configuration.
1727  *
1728  * @param dev
1729  *  Pointer to Ethernet device structure.
1730  * @param fc_conf
1731  *  Pointer to the flow control configuration.
1732  *
1733  * @return
1734  *  0 on success, negative error value otherwise.
1735  */
1736 static int
1737 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1738 {
1739         struct mrvl_priv *priv = dev->data->dev_private;
1740
1741         if (!priv)
1742                 return -EPERM;
1743
1744         if (fc_conf->high_water ||
1745             fc_conf->low_water ||
1746             fc_conf->pause_time ||
1747             fc_conf->mac_ctrl_frame_fwd ||
1748             fc_conf->autoneg) {
1749                 MRVL_LOG(ERR, "Flowctrl parameter is not supported");
1750
1751                 return -EINVAL;
1752         }
1753
1754         if (fc_conf->mode == RTE_FC_NONE ||
1755             fc_conf->mode == RTE_FC_RX_PAUSE) {
1756                 int ret, en;
1757
1758                 en = fc_conf->mode == RTE_FC_NONE ? 0 : 1;
1759                 ret = pp2_ppio_set_rx_pause(priv->ppio, en);
1760                 if (ret)
1761                         MRVL_LOG(ERR,
1762                                 "Failed to change flowctrl on RX side");
1763
1764                 return ret;
1765         }
1766
1767         return 0;
1768 }
1769
1770 /**
1771  * Update RSS hash configuration
1772  *
1773  * @param dev
1774  *   Pointer to Ethernet device structure.
1775  * @param rss_conf
1776  *   Pointer to RSS configuration.
1777  *
1778  * @return
1779  *   0 on success, negative error value otherwise.
1780  */
1781 static int
1782 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1783                      struct rte_eth_rss_conf *rss_conf)
1784 {
1785         struct mrvl_priv *priv = dev->data->dev_private;
1786
1787         if (priv->isolated)
1788                 return -ENOTSUP;
1789
1790         return mrvl_configure_rss(priv, rss_conf);
1791 }
1792
1793 /**
1794  * DPDK callback to get RSS hash configuration.
1795  *
1796  * @param dev
1797  *   Pointer to Ethernet device structure.
1798  * @rss_conf
1799  *   Pointer to RSS configuration.
1800  *
1801  * @return
1802  *   Always 0.
1803  */
1804 static int
1805 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1806                        struct rte_eth_rss_conf *rss_conf)
1807 {
1808         struct mrvl_priv *priv = dev->data->dev_private;
1809         enum pp2_ppio_hash_type hash_type =
1810                 priv->ppio_params.inqs_params.hash_type;
1811
1812         rss_conf->rss_key = NULL;
1813
1814         if (hash_type == PP2_PPIO_HASH_T_NONE)
1815                 rss_conf->rss_hf = 0;
1816         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1817                 rss_conf->rss_hf = ETH_RSS_IPV4;
1818         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1819                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1820         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1821                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1822
1823         return 0;
1824 }
1825
1826 /**
1827  * DPDK callback to get rte_flow callbacks.
1828  *
1829  * @param dev
1830  *   Pointer to the device structure.
1831  * @param filer_type
1832  *   Flow filter type.
1833  * @param filter_op
1834  *   Flow filter operation.
1835  * @param arg
1836  *   Pointer to pass the flow ops.
1837  *
1838  * @return
1839  *   0 on success, negative error value otherwise.
1840  */
1841 static int
1842 mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
1843                      enum rte_filter_type filter_type,
1844                      enum rte_filter_op filter_op, void *arg)
1845 {
1846         switch (filter_type) {
1847         case RTE_ETH_FILTER_GENERIC:
1848                 if (filter_op != RTE_ETH_FILTER_GET)
1849                         return -EINVAL;
1850                 *(const void **)arg = &mrvl_flow_ops;
1851                 return 0;
1852         default:
1853                 MRVL_LOG(WARNING, "Filter type (%d) not supported",
1854                                 filter_type);
1855                 return -EINVAL;
1856         }
1857 }
1858
1859 static const struct eth_dev_ops mrvl_ops = {
1860         .dev_configure = mrvl_dev_configure,
1861         .dev_start = mrvl_dev_start,
1862         .dev_stop = mrvl_dev_stop,
1863         .dev_set_link_up = mrvl_dev_set_link_up,
1864         .dev_set_link_down = mrvl_dev_set_link_down,
1865         .dev_close = mrvl_dev_close,
1866         .link_update = mrvl_link_update,
1867         .promiscuous_enable = mrvl_promiscuous_enable,
1868         .allmulticast_enable = mrvl_allmulticast_enable,
1869         .promiscuous_disable = mrvl_promiscuous_disable,
1870         .allmulticast_disable = mrvl_allmulticast_disable,
1871         .mac_addr_remove = mrvl_mac_addr_remove,
1872         .mac_addr_add = mrvl_mac_addr_add,
1873         .mac_addr_set = mrvl_mac_addr_set,
1874         .mtu_set = mrvl_mtu_set,
1875         .stats_get = mrvl_stats_get,
1876         .stats_reset = mrvl_stats_reset,
1877         .xstats_get = mrvl_xstats_get,
1878         .xstats_reset = mrvl_xstats_reset,
1879         .xstats_get_names = mrvl_xstats_get_names,
1880         .dev_infos_get = mrvl_dev_infos_get,
1881         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
1882         .rxq_info_get = mrvl_rxq_info_get,
1883         .txq_info_get = mrvl_txq_info_get,
1884         .vlan_filter_set = mrvl_vlan_filter_set,
1885         .tx_queue_start = mrvl_tx_queue_start,
1886         .tx_queue_stop = mrvl_tx_queue_stop,
1887         .rx_queue_setup = mrvl_rx_queue_setup,
1888         .rx_queue_release = mrvl_rx_queue_release,
1889         .tx_queue_setup = mrvl_tx_queue_setup,
1890         .tx_queue_release = mrvl_tx_queue_release,
1891         .flow_ctrl_get = mrvl_flow_ctrl_get,
1892         .flow_ctrl_set = mrvl_flow_ctrl_set,
1893         .rss_hash_update = mrvl_rss_hash_update,
1894         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1895         .filter_ctrl = mrvl_eth_filter_ctrl,
1896 };
1897
1898 /**
1899  * Return packet type information and l3/l4 offsets.
1900  *
1901  * @param desc
1902  *   Pointer to the received packet descriptor.
1903  * @param l3_offset
1904  *   l3 packet offset.
1905  * @param l4_offset
1906  *   l4 packet offset.
1907  *
1908  * @return
1909  *   Packet type information.
1910  */
1911 static inline uint64_t
1912 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
1913                                     uint8_t *l3_offset, uint8_t *l4_offset)
1914 {
1915         enum pp2_inq_l3_type l3_type;
1916         enum pp2_inq_l4_type l4_type;
1917         enum pp2_inq_vlan_tag vlan_tag;
1918         uint64_t packet_type;
1919
1920         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
1921         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
1922         pp2_ppio_inq_desc_get_vlan_tag(desc, &vlan_tag);
1923
1924         packet_type = RTE_PTYPE_L2_ETHER;
1925
1926         switch (vlan_tag) {
1927         case PP2_INQ_VLAN_TAG_SINGLE:
1928                 packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
1929                 break;
1930         case PP2_INQ_VLAN_TAG_DOUBLE:
1931         case PP2_INQ_VLAN_TAG_TRIPLE:
1932                 packet_type |= RTE_PTYPE_L2_ETHER_QINQ;
1933                 break;
1934         default:
1935                 break;
1936         }
1937
1938         switch (l3_type) {
1939         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
1940                 packet_type |= RTE_PTYPE_L3_IPV4;
1941                 break;
1942         case PP2_INQ_L3_TYPE_IPV4_OK:
1943                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
1944                 break;
1945         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
1946                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
1947                 break;
1948         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
1949                 packet_type |= RTE_PTYPE_L3_IPV6;
1950                 break;
1951         case PP2_INQ_L3_TYPE_IPV6_EXT:
1952                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
1953                 break;
1954         case PP2_INQ_L3_TYPE_ARP:
1955                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
1956                 /*
1957                  * In case of ARP l4_offset is set to wrong value.
1958                  * Set it to proper one so that later on mbuf->l3_len can be
1959                  * calculated subtracting l4_offset and l3_offset.
1960                  */
1961                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
1962                 break;
1963         default:
1964                 MRVL_LOG(DEBUG, "Failed to recognise l3 packet type");
1965                 break;
1966         }
1967
1968         switch (l4_type) {
1969         case PP2_INQ_L4_TYPE_TCP:
1970                 packet_type |= RTE_PTYPE_L4_TCP;
1971                 break;
1972         case PP2_INQ_L4_TYPE_UDP:
1973                 packet_type |= RTE_PTYPE_L4_UDP;
1974                 break;
1975         default:
1976                 MRVL_LOG(DEBUG, "Failed to recognise l4 packet type");
1977                 break;
1978         }
1979
1980         return packet_type;
1981 }
1982
1983 /**
1984  * Get offload information from the received packet descriptor.
1985  *
1986  * @param desc
1987  *   Pointer to the received packet descriptor.
1988  *
1989  * @return
1990  *   Mbuf offload flags.
1991  */
1992 static inline uint64_t
1993 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
1994 {
1995         uint64_t flags;
1996         enum pp2_inq_desc_status status;
1997
1998         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
1999         if (unlikely(status != PP2_DESC_ERR_OK))
2000                 flags = PKT_RX_IP_CKSUM_BAD;
2001         else
2002                 flags = PKT_RX_IP_CKSUM_GOOD;
2003
2004         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
2005         if (unlikely(status != PP2_DESC_ERR_OK))
2006                 flags |= PKT_RX_L4_CKSUM_BAD;
2007         else
2008                 flags |= PKT_RX_L4_CKSUM_GOOD;
2009
2010         return flags;
2011 }
2012
2013 /**
2014  * DPDK callback for receive.
2015  *
2016  * @param rxq
2017  *   Generic pointer to the receive queue.
2018  * @param rx_pkts
2019  *   Array to store received packets.
2020  * @param nb_pkts
2021  *   Maximum number of packets in array.
2022  *
2023  * @return
2024  *   Number of packets successfully received.
2025  */
2026 static uint16_t
2027 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2028 {
2029         struct mrvl_rxq *q = rxq;
2030         struct pp2_ppio_desc descs[nb_pkts];
2031         struct pp2_bpool *bpool;
2032         int i, ret, rx_done = 0;
2033         int num;
2034         struct pp2_hif *hif;
2035         unsigned int core_id = rte_lcore_id();
2036
2037         hif = mrvl_get_hif(q->priv, core_id);
2038
2039         if (unlikely(!q->priv->ppio || !hif))
2040                 return 0;
2041
2042         bpool = q->priv->bpool;
2043
2044         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
2045                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
2046         if (unlikely(ret < 0)) {
2047                 MRVL_LOG(ERR, "Failed to receive packets");
2048                 return 0;
2049         }
2050         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
2051
2052         for (i = 0; i < nb_pkts; i++) {
2053                 struct rte_mbuf *mbuf;
2054                 uint8_t l3_offset, l4_offset;
2055                 enum pp2_inq_desc_status status;
2056                 uint64_t addr;
2057
2058                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2059                         struct pp2_ppio_desc *pref_desc;
2060                         u64 pref_addr;
2061
2062                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
2063                         pref_addr = cookie_addr_high |
2064                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
2065                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
2066                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
2067                 }
2068
2069                 addr = cookie_addr_high |
2070                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
2071                 mbuf = (struct rte_mbuf *)addr;
2072                 rte_pktmbuf_reset(mbuf);
2073
2074                 /* drop packet in case of mac, overrun or resource error */
2075                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
2076                 if (unlikely(status != PP2_DESC_ERR_OK)) {
2077                         struct pp2_buff_inf binf = {
2078                                 .addr = rte_mbuf_data_iova_default(mbuf),
2079                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
2080                         };
2081
2082                         pp2_bpool_put_buff(hif, bpool, &binf);
2083                         mrvl_port_bpool_size
2084                                 [bpool->pp2_id][bpool->id][core_id]++;
2085                         q->drop_mac++;
2086                         continue;
2087                 }
2088
2089                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
2090                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
2091                 mbuf->data_len = mbuf->pkt_len;
2092                 mbuf->port = q->port_id;
2093                 mbuf->packet_type =
2094                         mrvl_desc_to_packet_type_and_offset(&descs[i],
2095                                                             &l3_offset,
2096                                                             &l4_offset);
2097                 mbuf->l2_len = l3_offset;
2098                 mbuf->l3_len = l4_offset - l3_offset;
2099
2100                 if (likely(q->cksum_enabled))
2101                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
2102
2103                 rx_pkts[rx_done++] = mbuf;
2104                 q->bytes_recv += mbuf->pkt_len;
2105         }
2106
2107         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
2108                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
2109
2110                 if (unlikely(num <= q->priv->bpool_min_size ||
2111                              (!rx_done && num < q->priv->bpool_init_size))) {
2112                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
2113                         if (ret)
2114                                 MRVL_LOG(ERR, "Failed to fill bpool");
2115                 } else if (unlikely(num > q->priv->bpool_max_size)) {
2116                         int i;
2117                         int pkt_to_remove = num - q->priv->bpool_init_size;
2118                         struct rte_mbuf *mbuf;
2119                         struct pp2_buff_inf buff;
2120
2121                         MRVL_LOG(DEBUG,
2122                                 "port-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)",
2123                                 bpool->pp2_id, q->priv->ppio->port_id,
2124                                 bpool->id, pkt_to_remove, num,
2125                                 q->priv->bpool_init_size);
2126
2127                         for (i = 0; i < pkt_to_remove; i++) {
2128                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
2129                                 if (ret)
2130                                         break;
2131                                 mbuf = (struct rte_mbuf *)
2132                                         (cookie_addr_high | buff.cookie);
2133                                 rte_pktmbuf_free(mbuf);
2134                         }
2135                         mrvl_port_bpool_size
2136                                 [bpool->pp2_id][bpool->id][core_id] -= i;
2137                 }
2138                 rte_spinlock_unlock(&q->priv->lock);
2139         }
2140
2141         return rx_done;
2142 }
2143
2144 /**
2145  * Prepare offload information.
2146  *
2147  * @param ol_flags
2148  *   Offload flags.
2149  * @param packet_type
2150  *   Packet type bitfield.
2151  * @param l3_type
2152  *   Pointer to the pp2_ouq_l3_type structure.
2153  * @param l4_type
2154  *   Pointer to the pp2_outq_l4_type structure.
2155  * @param gen_l3_cksum
2156  *   Will be set to 1 in case l3 checksum is computed.
2157  * @param l4_cksum
2158  *   Will be set to 1 in case l4 checksum is computed.
2159  *
2160  * @return
2161  *   0 on success, negative error value otherwise.
2162  */
2163 static inline int
2164 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
2165                         enum pp2_outq_l3_type *l3_type,
2166                         enum pp2_outq_l4_type *l4_type,
2167                         int *gen_l3_cksum,
2168                         int *gen_l4_cksum)
2169 {
2170         /*
2171          * Based on ol_flags prepare information
2172          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
2173          * for offloading.
2174          */
2175         if (ol_flags & PKT_TX_IPV4) {
2176                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
2177                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
2178         } else if (ol_flags & PKT_TX_IPV6) {
2179                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
2180                 /* no checksum for ipv6 header */
2181                 *gen_l3_cksum = 0;
2182         } else {
2183                 /* if something different then stop processing */
2184                 return -1;
2185         }
2186
2187         ol_flags &= PKT_TX_L4_MASK;
2188         if ((packet_type & RTE_PTYPE_L4_TCP) &&
2189             ol_flags == PKT_TX_TCP_CKSUM) {
2190                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
2191                 *gen_l4_cksum = 1;
2192         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
2193                    ol_flags == PKT_TX_UDP_CKSUM) {
2194                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
2195                 *gen_l4_cksum = 1;
2196         } else {
2197                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
2198                 /* no checksum for other type */
2199                 *gen_l4_cksum = 0;
2200         }
2201
2202         return 0;
2203 }
2204
2205 /**
2206  * Release already sent buffers to bpool (buffer-pool).
2207  *
2208  * @param ppio
2209  *   Pointer to the port structure.
2210  * @param hif
2211  *   Pointer to the MUSDK hardware interface.
2212  * @param sq
2213  *   Pointer to the shadow queue.
2214  * @param qid
2215  *   Queue id number.
2216  * @param force
2217  *   Force releasing packets.
2218  */
2219 static inline void
2220 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
2221                        unsigned int core_id, struct mrvl_shadow_txq *sq,
2222                        int qid, int force)
2223 {
2224         struct buff_release_entry *entry;
2225         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
2226         int i;
2227
2228         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
2229
2230         sq->num_to_release += nb_done;
2231
2232         if (likely(!force &&
2233                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
2234                 return;
2235
2236         nb_done = sq->num_to_release;
2237         sq->num_to_release = 0;
2238
2239         for (i = 0; i < nb_done; i++) {
2240                 entry = &sq->ent[sq->tail + num];
2241                 if (unlikely(!entry->buff.addr)) {
2242                         MRVL_LOG(ERR,
2243                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!",
2244                                 sq->tail, (u64)entry->buff.cookie,
2245                                 (u64)entry->buff.addr);
2246                         skip_bufs = 1;
2247                         goto skip;
2248                 }
2249
2250                 if (unlikely(!entry->bpool)) {
2251                         struct rte_mbuf *mbuf;
2252
2253                         mbuf = (struct rte_mbuf *)
2254                                (cookie_addr_high | entry->buff.cookie);
2255                         rte_pktmbuf_free(mbuf);
2256                         skip_bufs = 1;
2257                         goto skip;
2258                 }
2259
2260                 mrvl_port_bpool_size
2261                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
2262                 num++;
2263                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
2264                         goto skip;
2265                 continue;
2266 skip:
2267                 if (likely(num))
2268                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2269                 num += skip_bufs;
2270                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2271                 sq->size -= num;
2272                 num = 0;
2273                 skip_bufs = 0;
2274         }
2275
2276         if (likely(num)) {
2277                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2278                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2279                 sq->size -= num;
2280         }
2281 }
2282
2283 /**
2284  * DPDK callback for transmit.
2285  *
2286  * @param txq
2287  *   Generic pointer transmit queue.
2288  * @param tx_pkts
2289  *   Packets to transmit.
2290  * @param nb_pkts
2291  *   Number of packets in array.
2292  *
2293  * @return
2294  *   Number of packets successfully transmitted.
2295  */
2296 static uint16_t
2297 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2298 {
2299         struct mrvl_txq *q = txq;
2300         struct mrvl_shadow_txq *sq;
2301         struct pp2_hif *hif;
2302         struct pp2_ppio_desc descs[nb_pkts];
2303         unsigned int core_id = rte_lcore_id();
2304         int i, ret, bytes_sent = 0;
2305         uint16_t num, sq_free_size;
2306         uint64_t addr;
2307
2308         hif = mrvl_get_hif(q->priv, core_id);
2309         sq = &q->shadow_txqs[core_id];
2310
2311         if (unlikely(!q->priv->ppio || !hif))
2312                 return 0;
2313
2314         if (sq->size)
2315                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2316                                        sq, q->queue_id, 0);
2317
2318         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2319         if (unlikely(nb_pkts > sq_free_size)) {
2320                 MRVL_LOG(DEBUG,
2321                         "No room in shadow queue for %d packets! %d packets will be sent.",
2322                         nb_pkts, sq_free_size);
2323                 nb_pkts = sq_free_size;
2324         }
2325
2326         for (i = 0; i < nb_pkts; i++) {
2327                 struct rte_mbuf *mbuf = tx_pkts[i];
2328                 int gen_l3_cksum, gen_l4_cksum;
2329                 enum pp2_outq_l3_type l3_type;
2330                 enum pp2_outq_l4_type l4_type;
2331
2332                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2333                         struct rte_mbuf *pref_pkt_hdr;
2334
2335                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2336                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2337                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2338                 }
2339
2340                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
2341                 sq->ent[sq->head].buff.addr =
2342                         rte_mbuf_data_iova_default(mbuf);
2343                 sq->ent[sq->head].bpool =
2344                         (unlikely(mbuf->port >= RTE_MAX_ETHPORTS ||
2345                          mbuf->refcnt > 1)) ? NULL :
2346                          mrvl_port_to_bpool_lookup[mbuf->port];
2347                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
2348                 sq->size++;
2349
2350                 pp2_ppio_outq_desc_reset(&descs[i]);
2351                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
2352                                                  rte_pktmbuf_iova(mbuf));
2353                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
2354                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
2355                                                rte_pktmbuf_pkt_len(mbuf));
2356
2357                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2358                 /*
2359                  * in case unsupported ol_flags were passed
2360                  * do not update descriptor offload information
2361                  */
2362                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
2363                                               &l3_type, &l4_type, &gen_l3_cksum,
2364                                               &gen_l4_cksum);
2365                 if (unlikely(ret))
2366                         continue;
2367
2368                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2369                                                   mbuf->l2_len,
2370                                                   mbuf->l2_len + mbuf->l3_len,
2371                                                   gen_l3_cksum, gen_l4_cksum);
2372         }
2373
2374         num = nb_pkts;
2375         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2376         /* number of packets that were not sent */
2377         if (unlikely(num > nb_pkts)) {
2378                 for (i = nb_pkts; i < num; i++) {
2379                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2380                                 MRVL_PP2_TX_SHADOWQ_MASK;
2381                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
2382                         bytes_sent -=
2383                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2384                 }
2385                 sq->size -= num - nb_pkts;
2386         }
2387
2388         q->bytes_sent += bytes_sent;
2389
2390         return nb_pkts;
2391 }
2392
2393 /**
2394  * Initialize packet processor.
2395  *
2396  * @return
2397  *   0 on success, negative error value otherwise.
2398  */
2399 static int
2400 mrvl_init_pp2(void)
2401 {
2402         struct pp2_init_params init_params;
2403
2404         memset(&init_params, 0, sizeof(init_params));
2405         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
2406         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
2407         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
2408
2409         return pp2_init(&init_params);
2410 }
2411
2412 /**
2413  * Deinitialize packet processor.
2414  *
2415  * @return
2416  *   0 on success, negative error value otherwise.
2417  */
2418 static void
2419 mrvl_deinit_pp2(void)
2420 {
2421         pp2_deinit();
2422 }
2423
2424 /**
2425  * Create private device structure.
2426  *
2427  * @param dev_name
2428  *   Pointer to the port name passed in the initialization parameters.
2429  *
2430  * @return
2431  *   Pointer to the newly allocated private device structure.
2432  */
2433 static struct mrvl_priv *
2434 mrvl_priv_create(const char *dev_name)
2435 {
2436         struct pp2_bpool_params bpool_params;
2437         char match[MRVL_MATCH_LEN];
2438         struct mrvl_priv *priv;
2439         int ret, bpool_bit;
2440
2441         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
2442         if (!priv)
2443                 return NULL;
2444
2445         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
2446                                        &priv->pp_id, &priv->ppio_id);
2447         if (ret)
2448                 goto out_free_priv;
2449
2450         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
2451                                      PP2_BPOOL_NUM_POOLS);
2452         if (bpool_bit < 0)
2453                 goto out_free_priv;
2454         priv->bpool_bit = bpool_bit;
2455
2456         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
2457                  priv->bpool_bit);
2458         memset(&bpool_params, 0, sizeof(bpool_params));
2459         bpool_params.match = match;
2460         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
2461         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
2462         if (ret)
2463                 goto out_clear_bpool_bit;
2464
2465         priv->ppio_params.type = PP2_PPIO_T_NIC;
2466         rte_spinlock_init(&priv->lock);
2467
2468         return priv;
2469 out_clear_bpool_bit:
2470         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2471 out_free_priv:
2472         rte_free(priv);
2473         return NULL;
2474 }
2475
2476 /**
2477  * Create device representing Ethernet port.
2478  *
2479  * @param name
2480  *   Pointer to the port's name.
2481  *
2482  * @return
2483  *   0 on success, negative error value otherwise.
2484  */
2485 static int
2486 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
2487 {
2488         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
2489         struct rte_eth_dev *eth_dev;
2490         struct mrvl_priv *priv;
2491         struct ifreq req;
2492
2493         eth_dev = rte_eth_dev_allocate(name);
2494         if (!eth_dev)
2495                 return -ENOMEM;
2496
2497         priv = mrvl_priv_create(name);
2498         if (!priv) {
2499                 ret = -ENOMEM;
2500                 goto out_free_dev;
2501         }
2502
2503         eth_dev->data->mac_addrs =
2504                 rte_zmalloc("mac_addrs",
2505                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2506         if (!eth_dev->data->mac_addrs) {
2507                 MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
2508                 ret = -ENOMEM;
2509                 goto out_free_priv;
2510         }
2511
2512         memset(&req, 0, sizeof(req));
2513         strcpy(req.ifr_name, name);
2514         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2515         if (ret)
2516                 goto out_free_mac;
2517
2518         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2519                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2520
2521         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2522         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
2523         eth_dev->data->kdrv = RTE_KDRV_NONE;
2524         eth_dev->data->dev_private = priv;
2525         eth_dev->device = &vdev->device;
2526         eth_dev->dev_ops = &mrvl_ops;
2527
2528         rte_eth_dev_probing_finish(eth_dev);
2529         return 0;
2530 out_free_mac:
2531         rte_free(eth_dev->data->mac_addrs);
2532 out_free_dev:
2533         rte_eth_dev_release_port(eth_dev);
2534 out_free_priv:
2535         rte_free(priv);
2536
2537         return ret;
2538 }
2539
2540 /**
2541  * Cleanup previously created device representing Ethernet port.
2542  *
2543  * @param name
2544  *   Pointer to the port name.
2545  */
2546 static void
2547 mrvl_eth_dev_destroy(const char *name)
2548 {
2549         struct rte_eth_dev *eth_dev;
2550         struct mrvl_priv *priv;
2551
2552         eth_dev = rte_eth_dev_allocated(name);
2553         if (!eth_dev)
2554                 return;
2555
2556         priv = eth_dev->data->dev_private;
2557         pp2_bpool_deinit(priv->bpool);
2558         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2559         rte_free(priv);
2560         rte_free(eth_dev->data->mac_addrs);
2561         rte_eth_dev_release_port(eth_dev);
2562 }
2563
2564 /**
2565  * Callback used by rte_kvargs_process() during argument parsing.
2566  *
2567  * @param key
2568  *   Pointer to the parsed key (unused).
2569  * @param value
2570  *   Pointer to the parsed value.
2571  * @param extra_args
2572  *   Pointer to the extra arguments which contains address of the
2573  *   table of pointers to parsed interface names.
2574  *
2575  * @return
2576  *   Always 0.
2577  */
2578 static int
2579 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2580                  void *extra_args)
2581 {
2582         struct mrvl_ifnames *ifnames = extra_args;
2583
2584         ifnames->names[ifnames->idx++] = value;
2585
2586         return 0;
2587 }
2588
2589 /**
2590  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2591  */
2592 static void
2593 mrvl_deinit_hifs(void)
2594 {
2595         int i;
2596
2597         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
2598                 if (hifs[i])
2599                         pp2_hif_deinit(hifs[i]);
2600         }
2601         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
2602         memset(hifs, 0, sizeof(hifs));
2603 }
2604
2605 /**
2606  * DPDK callback to register the virtual device.
2607  *
2608  * @param vdev
2609  *   Pointer to the virtual device.
2610  *
2611  * @return
2612  *   0 on success, negative error value otherwise.
2613  */
2614 static int
2615 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2616 {
2617         struct rte_kvargs *kvlist;
2618         struct mrvl_ifnames ifnames;
2619         int ret = -EINVAL;
2620         uint32_t i, ifnum, cfgnum;
2621         const char *params;
2622
2623         params = rte_vdev_device_args(vdev);
2624         if (!params)
2625                 return -EINVAL;
2626
2627         kvlist = rte_kvargs_parse(params, valid_args);
2628         if (!kvlist)
2629                 return -EINVAL;
2630
2631         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2632         if (ifnum > RTE_DIM(ifnames.names))
2633                 goto out_free_kvlist;
2634
2635         ifnames.idx = 0;
2636         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2637                            mrvl_get_ifnames, &ifnames);
2638
2639
2640         /*
2641          * The below system initialization should be done only once,
2642          * on the first provided configuration file
2643          */
2644         if (!mrvl_qos_cfg) {
2645                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2646                 MRVL_LOG(INFO, "Parsing config file!");
2647                 if (cfgnum > 1) {
2648                         MRVL_LOG(ERR, "Cannot handle more than one config file!");
2649                         goto out_free_kvlist;
2650                 } else if (cfgnum == 1) {
2651                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2652                                            mrvl_get_qoscfg, &mrvl_qos_cfg);
2653                 }
2654         }
2655
2656         if (mrvl_dev_num)
2657                 goto init_devices;
2658
2659         MRVL_LOG(INFO, "Perform MUSDK initializations");
2660
2661         ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist);
2662         if (ret)
2663                 goto out_free_kvlist;
2664
2665         ret = mrvl_init_pp2();
2666         if (ret) {
2667                 MRVL_LOG(ERR, "Failed to init PP!");
2668                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2669                 goto out_free_kvlist;
2670         }
2671
2672         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2673         memset(mrvl_port_to_bpool_lookup, 0, sizeof(mrvl_port_to_bpool_lookup));
2674
2675         mrvl_lcore_first = RTE_MAX_LCORE;
2676         mrvl_lcore_last = 0;
2677
2678 init_devices:
2679         for (i = 0; i < ifnum; i++) {
2680                 MRVL_LOG(INFO, "Creating %s", ifnames.names[i]);
2681                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
2682                 if (ret)
2683                         goto out_cleanup;
2684         }
2685         mrvl_dev_num += ifnum;
2686
2687         rte_kvargs_free(kvlist);
2688
2689         return 0;
2690 out_cleanup:
2691         for (; i > 0; i--)
2692                 mrvl_eth_dev_destroy(ifnames.names[i]);
2693
2694         if (mrvl_dev_num == 0) {
2695                 mrvl_deinit_pp2();
2696                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2697         }
2698 out_free_kvlist:
2699         rte_kvargs_free(kvlist);
2700
2701         return ret;
2702 }
2703
2704 /**
2705  * DPDK callback to remove virtual device.
2706  *
2707  * @param vdev
2708  *   Pointer to the removed virtual device.
2709  *
2710  * @return
2711  *   0 on success, negative error value otherwise.
2712  */
2713 static int
2714 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2715 {
2716         int i;
2717         const char *name;
2718
2719         name = rte_vdev_device_name(vdev);
2720         if (!name)
2721                 return -EINVAL;
2722
2723         MRVL_LOG(INFO, "Removing %s", name);
2724
2725         RTE_ETH_FOREACH_DEV(i) { /* FIXME: removing all devices! */
2726                 char ifname[RTE_ETH_NAME_MAX_LEN];
2727
2728                 rte_eth_dev_get_name_by_port(i, ifname);
2729                 mrvl_eth_dev_destroy(ifname);
2730                 mrvl_dev_num--;
2731         }
2732
2733         if (mrvl_dev_num == 0) {
2734                 MRVL_LOG(INFO, "Perform MUSDK deinit");
2735                 mrvl_deinit_hifs();
2736                 mrvl_deinit_pp2();
2737                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2738         }
2739
2740         return 0;
2741 }
2742
2743 static struct rte_vdev_driver pmd_mrvl_drv = {
2744         .probe = rte_pmd_mrvl_probe,
2745         .remove = rte_pmd_mrvl_remove,
2746 };
2747
2748 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv);
2749 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2);
2750
2751 RTE_INIT(mrvl_init_log)
2752 {
2753         mrvl_logtype = rte_log_register("pmd.net.mvpp2");
2754         if (mrvl_logtype >= 0)
2755                 rte_log_set_level(mrvl_logtype, RTE_LOG_NOTICE);
2756 }