drivers/net/mvpp2/mrvl_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Marvell International Ltd.
   3  * Copyright(c) 2017 Semihalf.
   4  * All rights reserved.
   5  */
   6
   7 #include <rte_ethdev_driver.h>
   8 #include <rte_kvargs.h>
   9 #include <rte_log.h>
  10 #include <rte_malloc.h>
  11 #include <rte_bus_vdev.h>
  12
  13 /* Unluckily, container_of is defined by both DPDK and MUSDK,
  14  * we'll declare only one version.
  15  *
  16  * Note that it is not used in this PMD anyway.
  17  */
  18 #ifdef container_of
  19 #undef container_of
  20 #endif
  21
  22 #include <fcntl.h>
  23 #include <linux/ethtool.h>
  24 #include <linux/sockios.h>
  25 #include <net/if.h>
  26 #include <net/if_arp.h>
  27 #include <sys/ioctl.h>
  28 #include <sys/socket.h>
  29 #include <sys/stat.h>
  30 #include <sys/types.h>
  31
  32 #include <rte_mvep_common.h>
  33 #include "mrvl_ethdev.h"
  34 #include "mrvl_qos.h"
  35
  36 /* bitmask with reserved hifs */
  37 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  38 /* bitmask with reserved bpools */
  39 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  40 /* bitmask with reserved kernel RSS tables */
  41 #define MRVL_MUSDK_RSS_RESERVED 0x01
  42 /* maximum number of available hifs */
  43 #define MRVL_MUSDK_HIFS_MAX 9
  44
  45 /* prefetch shift */
  46 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  47
  48 /* TCAM has 25 entries reserved for uc/mc filter entries */
  49 #define MRVL_MAC_ADDRS_MAX 25
  50 #define MRVL_MATCH_LEN 16
  51 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  52 /* Maximum allowable packet size */
  53 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  54
  55 #define MRVL_IFACE_NAME_ARG "iface"
  56 #define MRVL_CFG_ARG "cfg"
  57
  58 #define MRVL_BURST_SIZE 64
  59
  60 #define MRVL_ARP_LENGTH 28
  61
  62 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  63
  64 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
  65 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
  66
  67 /** Port Rx offload capabilities */
  68 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
  69                           DEV_RX_OFFLOAD_JUMBO_FRAME | \
  70                           DEV_RX_OFFLOAD_CRC_STRIP | \
  71                           DEV_RX_OFFLOAD_CHECKSUM)
  72
  73 /** Port Tx offloads capabilities */
  74 #define MRVL_TX_OFFLOADS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
  75                           DEV_TX_OFFLOAD_UDP_CKSUM | \
  76                           DEV_TX_OFFLOAD_TCP_CKSUM)
  77
  78 static const char * const valid_args[] = {
  79         MRVL_IFACE_NAME_ARG,
  80         MRVL_CFG_ARG,
  81         NULL
  82 };
  83
  84 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
  85 static struct pp2_hif *hifs[RTE_MAX_LCORE];
  86 static int used_bpools[PP2_NUM_PKT_PROC] = {
  87         [0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED
  88 };
  89
  90 static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
  91 static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
  92 static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
  93
  94 int mrvl_logtype;
  95
  96 struct mrvl_ifnames {
  97         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
  98         int idx;
  99 };
 100
 101 /*
 102  * To use buffer harvesting based on loopback port shadow queue structure
 103  * was introduced for buffers information bookkeeping.
 104  *
 105  * Before sending the packet, related buffer information (pp2_buff_inf) is
 106  * stored in shadow queue. After packet is transmitted no longer used
 107  * packet buffer is released back to it's original hardware pool,
 108  * on condition it originated from interface.
 109  * In case it  was generated by application itself i.e: mbuf->port field is
 110  * 0xff then its released to software mempool.
 111  */
 112 struct mrvl_shadow_txq {
 113         int head;           /* write index - used when sending buffers */
 114         int tail;           /* read index - used when releasing buffers */
 115         u16 size;           /* queue occupied size */
 116         u16 num_to_release; /* number of buffers sent, that can be released */
 117         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 118 };
 119
 120 struct mrvl_rxq {
 121         struct mrvl_priv *priv;
 122         struct rte_mempool *mp;
 123         int queue_id;
 124         int port_id;
 125         int cksum_enabled;
 126         uint64_t bytes_recv;
 127         uint64_t drop_mac;
 128 };
 129
 130 struct mrvl_txq {
 131         struct mrvl_priv *priv;
 132         int queue_id;
 133         int port_id;
 134         uint64_t bytes_sent;
 135         struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE];
 136         int tx_deferred_start;
 137 };
 138
 139 static int mrvl_lcore_first;
 140 static int mrvl_lcore_last;
 141 static int mrvl_dev_num;
 142
 143 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
 144 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
 145                         struct pp2_hif *hif, unsigned int core_id,
 146                         struct mrvl_shadow_txq *sq, int qid, int force);
 147
 148 #define MRVL_XSTATS_TBL_ENTRY(name) { \
 149         #name, offsetof(struct pp2_ppio_statistics, name),      \
 150         sizeof(((struct pp2_ppio_statistics *)0)->name)         \
 151 }
 152
 153 /* Table with xstats data */
 154 static struct {
 155         const char *name;
 156         unsigned int offset;
 157         unsigned int size;
 158 } mrvl_xstats_tbl[] = {
 159         MRVL_XSTATS_TBL_ENTRY(rx_bytes),
 160         MRVL_XSTATS_TBL_ENTRY(rx_packets),
 161         MRVL_XSTATS_TBL_ENTRY(rx_unicast_packets),
 162         MRVL_XSTATS_TBL_ENTRY(rx_errors),
 163         MRVL_XSTATS_TBL_ENTRY(rx_fullq_dropped),
 164         MRVL_XSTATS_TBL_ENTRY(rx_bm_dropped),
 165         MRVL_XSTATS_TBL_ENTRY(rx_early_dropped),
 166         MRVL_XSTATS_TBL_ENTRY(rx_fifo_dropped),
 167         MRVL_XSTATS_TBL_ENTRY(rx_cls_dropped),
 168         MRVL_XSTATS_TBL_ENTRY(tx_bytes),
 169         MRVL_XSTATS_TBL_ENTRY(tx_packets),
 170         MRVL_XSTATS_TBL_ENTRY(tx_unicast_packets),
 171         MRVL_XSTATS_TBL_ENTRY(tx_errors)
 172 };
 173
 174 static inline int
 175 mrvl_get_bpool_size(int pp2_id, int pool_id)
 176 {
 177         int i;
 178         int size = 0;
 179
 180         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 181                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 182
 183         return size;
 184 }
 185
 186 static inline int
 187 mrvl_reserve_bit(int *bitmap, int max)
 188 {
 189         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 190
 191         if (n >= max)
 192                 return -1;
 193
 194         *bitmap |= 1 << n;
 195
 196         return n;
 197 }
 198
 199 static int
 200 mrvl_init_hif(int core_id)
 201 {
 202         struct pp2_hif_params params;
 203         char match[MRVL_MATCH_LEN];
 204         int ret;
 205
 206         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
 207         if (ret < 0) {
 208                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 209                 return ret;
 210         }
 211
 212         snprintf(match, sizeof(match), "hif-%d", ret);
 213         memset(&params, 0, sizeof(params));
 214         params.match = match;
 215         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
 216         ret = pp2_hif_init(&params, &hifs[core_id]);
 217         if (ret) {
 218                 MRVL_LOG(ERR, "Failed to initialize hif %d", core_id);
 219                 return ret;
 220         }
 221
 222         return 0;
 223 }
 224
 225 static inline struct pp2_hif*
 226 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
 227 {
 228         int ret;
 229
 230         if (likely(hifs[core_id] != NULL))
 231                 return hifs[core_id];
 232
 233         rte_spinlock_lock(&priv->lock);
 234
 235         ret = mrvl_init_hif(core_id);
 236         if (ret < 0) {
 237                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 238                 goto out;
 239         }
 240
 241         if (core_id < mrvl_lcore_first)
 242                 mrvl_lcore_first = core_id;
 243
 244         if (core_id > mrvl_lcore_last)
 245                 mrvl_lcore_last = core_id;
 246 out:
 247         rte_spinlock_unlock(&priv->lock);
 248
 249         return hifs[core_id];
 250 }
 251
 252 /**
 253  * Configure rss based on dpdk rss configuration.
 254  *
 255  * @param priv
 256  *   Pointer to private structure.
 257  * @param rss_conf
 258  *   Pointer to RSS configuration.
 259  *
 260  * @return
 261  *   0 on success, negative error value otherwise.
 262  */
 263 static int
 264 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 265 {
 266         if (rss_conf->rss_key)
 267                 MRVL_LOG(WARNING, "Changing hash key is not supported");
 268
 269         if (rss_conf->rss_hf == 0) {
 270                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 271         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 272                 priv->ppio_params.inqs_params.hash_type =
 273                         PP2_PPIO_HASH_T_2_TUPLE;
 274         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 275                 priv->ppio_params.inqs_params.hash_type =
 276                         PP2_PPIO_HASH_T_5_TUPLE;
 277                 priv->rss_hf_tcp = 1;
 278         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 279                 priv->ppio_params.inqs_params.hash_type =
 280                         PP2_PPIO_HASH_T_5_TUPLE;
 281                 priv->rss_hf_tcp = 0;
 282         } else {
 283                 return -EINVAL;
 284         }
 285
 286         return 0;
 287 }
 288
 289 /**
 290  * Ethernet device configuration.
 291  *
 292  * Prepare the driver for a given number of TX and RX queues and
 293  * configure RSS.
 294  *
 295  * @param dev
 296  *   Pointer to Ethernet device structure.
 297  *
 298  * @return
 299  *   0 on success, negative error value otherwise.
 300  */
 301 static int
 302 mrvl_dev_configure(struct rte_eth_dev *dev)
 303 {
 304         struct mrvl_priv *priv = dev->data->dev_private;
 305         int ret;
 306
 307         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 308             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 309                 MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
 310                         dev->data->dev_conf.rxmode.mq_mode);
 311                 return -EINVAL;
 312         }
 313
 314         /* KEEP_CRC offload flag is not supported by PMD
 315          * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
 316          */
 317         if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads)) {
 318                 MRVL_LOG(INFO, "L2 CRC stripping is always enabled in hw");
 319                 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
 320         }
 321
 322         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 323                 MRVL_LOG(INFO, "Split headers not supported");
 324                 return -EINVAL;
 325         }
 326
 327         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 328                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 329                                  ETHER_HDR_LEN - ETHER_CRC_LEN;
 330
 331         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 332                                   dev->data->nb_rx_queues);
 333         if (ret < 0)
 334                 return ret;
 335
 336         ret = mrvl_configure_txqs(priv, dev->data->port_id,
 337                                   dev->data->nb_tx_queues);
 338         if (ret < 0)
 339                 return ret;
 340
 341         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 342         priv->ppio_params.maintain_stats = 1;
 343         priv->nb_rx_queues = dev->data->nb_rx_queues;
 344
 345         if (dev->data->nb_rx_queues == 1 &&
 346             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 347                 MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
 348                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 349
 350                 return 0;
 351         }
 352
 353         return mrvl_configure_rss(priv,
 354                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
 355 }
 356
 357 /**
 358  * DPDK callback to change the MTU.
 359  *
 360  * Setting the MTU affects hardware MRU (packets larger than the MRU
 361  * will be dropped).
 362  *
 363  * @param dev
 364  *   Pointer to Ethernet device structure.
 365  * @param mtu
 366  *   New MTU.
 367  *
 368  * @return
 369  *   0 on success, negative error value otherwise.
 370  */
 371 static int
 372 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 373 {
 374         struct mrvl_priv *priv = dev->data->dev_private;
 375         /* extra MV_MH_SIZE bytes are required for Marvell tag */
 376         uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
 377         int ret;
 378
 379         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
 380                 return -EINVAL;
 381
 382         if (!priv->ppio)
 383                 return 0;
 384
 385         ret = pp2_ppio_set_mru(priv->ppio, mru);
 386         if (ret)
 387                 return ret;
 388
 389         return pp2_ppio_set_mtu(priv->ppio, mtu);
 390 }
 391
 392 /**
 393  * DPDK callback to bring the link up.
 394  *
 395  * @param dev
 396  *   Pointer to Ethernet device structure.
 397  *
 398  * @return
 399  *   0 on success, negative error value otherwise.
 400  */
 401 static int
 402 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 403 {
 404         struct mrvl_priv *priv = dev->data->dev_private;
 405         int ret;
 406
 407         if (!priv->ppio)
 408                 return -EPERM;
 409
 410         ret = pp2_ppio_enable(priv->ppio);
 411         if (ret)
 412                 return ret;
 413
 414         /*
 415          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 416          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 417          * PP2_TRAFFIC_INGRESS_EGRESS.
 418          *
 419          * Set mtu to default DPDK value here.
 420          */
 421         ret = mrvl_mtu_set(dev, dev->data->mtu);
 422         if (ret)
 423                 pp2_ppio_disable(priv->ppio);
 424
 425         return ret;
 426 }
 427
 428 /**
 429  * DPDK callback to bring the link down.
 430  *
 431  * @param dev
 432  *   Pointer to Ethernet device structure.
 433  *
 434  * @return
 435  *   0 on success, negative error value otherwise.
 436  */
 437 static int
 438 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 439 {
 440         struct mrvl_priv *priv = dev->data->dev_private;
 441
 442         if (!priv->ppio)
 443                 return -EPERM;
 444
 445         return pp2_ppio_disable(priv->ppio);
 446 }
 447
 448 /**
 449  * DPDK callback to start tx queue.
 450  *
 451  * @param dev
 452  *   Pointer to Ethernet device structure.
 453  * @param queue_id
 454  *   Transmit queue index.
 455  *
 456  * @return
 457  *   0 on success, negative error value otherwise.
 458  */
 459 static int
 460 mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
 461 {
 462         struct mrvl_priv *priv = dev->data->dev_private;
 463         int ret;
 464
 465         if (!priv)
 466                 return -EPERM;
 467
 468         /* passing 1 enables given tx queue */
 469         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1);
 470         if (ret) {
 471                 MRVL_LOG(ERR, "Failed to start txq %d", queue_id);
 472                 return ret;
 473         }
 474
 475         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 476
 477         return 0;
 478 }
 479
 480 /**
 481  * DPDK callback to stop tx queue.
 482  *
 483  * @param dev
 484  *   Pointer to Ethernet device structure.
 485  * @param queue_id
 486  *   Transmit queue index.
 487  *
 488  * @return
 489  *   0 on success, negative error value otherwise.
 490  */
 491 static int
 492 mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
 493 {
 494         struct mrvl_priv *priv = dev->data->dev_private;
 495         int ret;
 496
 497         if (!priv->ppio)
 498                 return -EPERM;
 499
 500         /* passing 0 disables given tx queue */
 501         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0);
 502         if (ret) {
 503                 MRVL_LOG(ERR, "Failed to stop txq %d", queue_id);
 504                 return ret;
 505         }
 506
 507         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 508
 509         return 0;
 510 }
 511
 512 /**
 513  * DPDK callback to start the device.
 514  *
 515  * @param dev
 516  *   Pointer to Ethernet device structure.
 517  *
 518  * @return
 519  *   0 on success, negative errno value on failure.
 520  */
 521 static int
 522 mrvl_dev_start(struct rte_eth_dev *dev)
 523 {
 524         struct mrvl_priv *priv = dev->data->dev_private;
 525         char match[MRVL_MATCH_LEN];
 526         int ret = 0, i, def_init_size;
 527
 528         snprintf(match, sizeof(match), "ppio-%d:%d",
 529                  priv->pp_id, priv->ppio_id);
 530         priv->ppio_params.match = match;
 531
 532         /*
 533          * Calculate the minimum bpool size for refill feature as follows:
 534          * 2 default burst sizes multiply by number of rx queues.
 535          * If the bpool size will be below this value, new buffers will
 536          * be added to the pool.
 537          */
 538         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 539
 540         /* In case initial bpool size configured in queues setup is
 541          * smaller than minimum size add more buffers
 542          */
 543         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
 544         if (priv->bpool_init_size < def_init_size) {
 545                 int buffs_to_add = def_init_size - priv->bpool_init_size;
 546
 547                 priv->bpool_init_size += buffs_to_add;
 548                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
 549                 if (ret)
 550                         MRVL_LOG(ERR, "Failed to add buffers to bpool");
 551         }
 552
 553         /*
 554          * Calculate the maximum bpool size for refill feature as follows:
 555          * maximum number of descriptors in rx queue multiply by number
 556          * of rx queues plus minimum bpool size.
 557          * In case the bpool size will exceed this value, superfluous buffers
 558          * will be removed
 559          */
 560         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
 561                                 priv->bpool_min_size;
 562
 563         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 564         if (ret) {
 565                 MRVL_LOG(ERR, "Failed to init ppio");
 566                 return ret;
 567         }
 568
 569         /*
 570          * In case there are some some stale uc/mc mac addresses flush them
 571          * here. It cannot be done during mrvl_dev_close() as port information
 572          * is already gone at that point (due to pp2_ppio_deinit() in
 573          * mrvl_dev_stop()).
 574          */
 575         if (!priv->uc_mc_flushed) {
 576                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 577                 if (ret) {
 578                         MRVL_LOG(ERR,
 579                                 "Failed to flush uc/mc filter list");
 580                         goto out;
 581                 }
 582                 priv->uc_mc_flushed = 1;
 583         }
 584
 585         if (!priv->vlan_flushed) {
 586                 ret = pp2_ppio_flush_vlan(priv->ppio);
 587                 if (ret) {
 588                         MRVL_LOG(ERR, "Failed to flush vlan list");
 589                         /*
 590                          * TODO
 591                          * once pp2_ppio_flush_vlan() is supported jump to out
 592                          * goto out;
 593                          */
 594                 }
 595                 priv->vlan_flushed = 1;
 596         }
 597
 598         /* For default QoS config, don't start classifier. */
 599         if (mrvl_qos_cfg) {
 600                 ret = mrvl_start_qos_mapping(priv);
 601                 if (ret) {
 602                         MRVL_LOG(ERR, "Failed to setup QoS mapping");
 603                         goto out;
 604                 }
 605         }
 606
 607         ret = mrvl_dev_set_link_up(dev);
 608         if (ret) {
 609                 MRVL_LOG(ERR, "Failed to set link up");
 610                 goto out;
 611         }
 612
 613         /* start tx queues */
 614         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 615                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 616
 617                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
 618
 619                 if (!txq->tx_deferred_start)
 620                         continue;
 621
 622                 /*
 623                  * All txqs are started by default. Stop them
 624                  * so that tx_deferred_start works as expected.
 625                  */
 626                 ret = mrvl_tx_queue_stop(dev, i);
 627                 if (ret)
 628                         goto out;
 629         }
 630
 631         return 0;
 632 out:
 633         MRVL_LOG(ERR, "Failed to start device");
 634         pp2_ppio_deinit(priv->ppio);
 635         return ret;
 636 }
 637
 638 /**
 639  * Flush receive queues.
 640  *
 641  * @param dev
 642  *   Pointer to Ethernet device structure.
 643  */
 644 static void
 645 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 646 {
 647         int i;
 648
 649         MRVL_LOG(INFO, "Flushing rx queues");
 650         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 651                 int ret, num;
 652
 653                 do {
 654                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 655                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 656
 657                         num = MRVL_PP2_RXD_MAX;
 658                         ret = pp2_ppio_recv(q->priv->ppio,
 659                                             q->priv->rxq_map[q->queue_id].tc,
 660                                             q->priv->rxq_map[q->queue_id].inq,
 661                                             descs, (uint16_t *)&num);
 662                 } while (ret == 0 && num);
 663         }
 664 }
 665
 666 /**
 667  * Flush transmit shadow queues.
 668  *
 669  * @param dev
 670  *   Pointer to Ethernet device structure.
 671  */
 672 static void
 673 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 674 {
 675         int i, j;
 676         struct mrvl_txq *txq;
 677
 678         MRVL_LOG(INFO, "Flushing tx shadow queues");
 679         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 680                 txq = (struct mrvl_txq *)dev->data->tx_queues[i];
 681
 682                 for (j = 0; j < RTE_MAX_LCORE; j++) {
 683                         struct mrvl_shadow_txq *sq;
 684
 685                         if (!hifs[j])
 686                                 continue;
 687
 688                         sq = &txq->shadow_txqs[j];
 689                         mrvl_free_sent_buffers(txq->priv->ppio,
 690                                 hifs[j], j, sq, txq->queue_id, 1);
 691                         while (sq->tail != sq->head) {
 692                                 uint64_t addr = cookie_addr_high |
 693                                         sq->ent[sq->tail].buff.cookie;
 694                                 rte_pktmbuf_free(
 695                                         (struct rte_mbuf *)addr);
 696                                 sq->tail = (sq->tail + 1) &
 697                                             MRVL_PP2_TX_SHADOWQ_MASK;
 698                         }
 699                         memset(sq, 0, sizeof(*sq));
 700                 }
 701         }
 702 }
 703
 704 /**
 705  * Flush hardware bpool (buffer-pool).
 706  *
 707  * @param dev
 708  *   Pointer to Ethernet device structure.
 709  */
 710 static void
 711 mrvl_flush_bpool(struct rte_eth_dev *dev)
 712 {
 713         struct mrvl_priv *priv = dev->data->dev_private;
 714         struct pp2_hif *hif;
 715         uint32_t num;
 716         int ret;
 717         unsigned int core_id = rte_lcore_id();
 718
 719         if (core_id == LCORE_ID_ANY)
 720                 core_id = 0;
 721
 722         hif = mrvl_get_hif(priv, core_id);
 723
 724         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 725         if (ret) {
 726                 MRVL_LOG(ERR, "Failed to get bpool buffers number");
 727                 return;
 728         }
 729
 730         while (num--) {
 731                 struct pp2_buff_inf inf;
 732                 uint64_t addr;
 733
 734                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
 735                 if (ret)
 736                         break;
 737
 738                 addr = cookie_addr_high | inf.cookie;
 739                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 740         }
 741 }
 742
 743 /**
 744  * DPDK callback to stop the device.
 745  *
 746  * @param dev
 747  *   Pointer to Ethernet device structure.
 748  */
 749 static void
 750 mrvl_dev_stop(struct rte_eth_dev *dev)
 751 {
 752         struct mrvl_priv *priv = dev->data->dev_private;
 753
 754         mrvl_dev_set_link_down(dev);
 755         mrvl_flush_rx_queues(dev);
 756         mrvl_flush_tx_shadow_queues(dev);
 757         if (priv->cls_tbl) {
 758                 pp2_cls_tbl_deinit(priv->cls_tbl);
 759                 priv->cls_tbl = NULL;
 760         }
 761         if (priv->qos_tbl) {
 762                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 763                 priv->qos_tbl = NULL;
 764         }
 765         if (priv->ppio)
 766                 pp2_ppio_deinit(priv->ppio);
 767         priv->ppio = NULL;
 768
 769         /* policer must be released after ppio deinitialization */
 770         if (priv->policer) {
 771                 pp2_cls_plcr_deinit(priv->policer);
 772                 priv->policer = NULL;
 773         }
 774 }
 775
 776 /**
 777  * DPDK callback to close the device.
 778  *
 779  * @param dev
 780  *   Pointer to Ethernet device structure.
 781  */
 782 static void
 783 mrvl_dev_close(struct rte_eth_dev *dev)
 784 {
 785         struct mrvl_priv *priv = dev->data->dev_private;
 786         size_t i;
 787
 788         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 789                 struct pp2_ppio_tc_params *tc_params =
 790                         &priv->ppio_params.inqs_params.tcs_params[i];
 791
 792                 if (tc_params->inqs_params) {
 793                         rte_free(tc_params->inqs_params);
 794                         tc_params->inqs_params = NULL;
 795                 }
 796         }
 797
 798         mrvl_flush_bpool(dev);
 799 }
 800
 801 /**
 802  * DPDK callback to retrieve physical link information.
 803  *
 804  * @param dev
 805  *   Pointer to Ethernet device structure.
 806  * @param wait_to_complete
 807  *   Wait for request completion (ignored).
 808  *
 809  * @return
 810  *   0 on success, negative error value otherwise.
 811  */
 812 static int
 813 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 814 {
 815         /*
 816          * TODO
 817          * once MUSDK provides necessary API use it here
 818          */
 819         struct mrvl_priv *priv = dev->data->dev_private;
 820         struct ethtool_cmd edata;
 821         struct ifreq req;
 822         int ret, fd, link_up;
 823
 824         if (!priv->ppio)
 825                 return -EPERM;
 826
 827         edata.cmd = ETHTOOL_GSET;
 828
 829         strcpy(req.ifr_name, dev->data->name);
 830         req.ifr_data = (void *)&edata;
 831
 832         fd = socket(AF_INET, SOCK_DGRAM, 0);
 833         if (fd == -1)
 834                 return -EFAULT;
 835
 836         ret = ioctl(fd, SIOCETHTOOL, &req);
 837         if (ret == -1) {
 838                 close(fd);
 839                 return -EFAULT;
 840         }
 841
 842         close(fd);
 843
 844         switch (ethtool_cmd_speed(&edata)) {
 845         case SPEED_10:
 846                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 847                 break;
 848         case SPEED_100:
 849                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 850                 break;
 851         case SPEED_1000:
 852                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 853                 break;
 854         case SPEED_10000:
 855                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 856                 break;
 857         default:
 858                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 859         }
 860
 861         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 862                                                          ETH_LINK_HALF_DUPLEX;
 863         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 864                                                            ETH_LINK_FIXED;
 865         pp2_ppio_get_link_state(priv->ppio, &link_up);
 866         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
 867
 868         return 0;
 869 }
 870
 871 /**
 872  * DPDK callback to enable promiscuous mode.
 873  *
 874  * @param dev
 875  *   Pointer to Ethernet device structure.
 876  */
 877 static void
 878 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
 879 {
 880         struct mrvl_priv *priv = dev->data->dev_private;
 881         int ret;
 882
 883         if (!priv->ppio)
 884                 return;
 885
 886         if (priv->isolated)
 887                 return;
 888
 889         ret = pp2_ppio_set_promisc(priv->ppio, 1);
 890         if (ret)
 891                 MRVL_LOG(ERR, "Failed to enable promiscuous mode");
 892 }
 893
 894 /**
 895  * DPDK callback to enable allmulti mode.
 896  *
 897  * @param dev
 898  *   Pointer to Ethernet device structure.
 899  */
 900 static void
 901 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
 902 {
 903         struct mrvl_priv *priv = dev->data->dev_private;
 904         int ret;
 905
 906         if (!priv->ppio)
 907                 return;
 908
 909         if (priv->isolated)
 910                 return;
 911
 912         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
 913         if (ret)
 914                 MRVL_LOG(ERR, "Failed enable all-multicast mode");
 915 }
 916
 917 /**
 918  * DPDK callback to disable promiscuous mode.
 919  *
 920  * @param dev
 921  *   Pointer to Ethernet device structure.
 922  */
 923 static void
 924 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
 925 {
 926         struct mrvl_priv *priv = dev->data->dev_private;
 927         int ret;
 928
 929         if (!priv->ppio)
 930                 return;
 931
 932         ret = pp2_ppio_set_promisc(priv->ppio, 0);
 933         if (ret)
 934                 MRVL_LOG(ERR, "Failed to disable promiscuous mode");
 935 }
 936
 937 /**
 938  * DPDK callback to disable allmulticast mode.
 939  *
 940  * @param dev
 941  *   Pointer to Ethernet device structure.
 942  */
 943 static void
 944 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
 945 {
 946         struct mrvl_priv *priv = dev->data->dev_private;
 947         int ret;
 948
 949         if (!priv->ppio)
 950                 return;
 951
 952         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
 953         if (ret)
 954                 MRVL_LOG(ERR, "Failed to disable all-multicast mode");
 955 }
 956
 957 /**
 958  * DPDK callback to remove a MAC address.
 959  *
 960  * @param dev
 961  *   Pointer to Ethernet device structure.
 962  * @param index
 963  *   MAC address index.
 964  */
 965 static void
 966 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 967 {
 968         struct mrvl_priv *priv = dev->data->dev_private;
 969         char buf[ETHER_ADDR_FMT_SIZE];
 970         int ret;
 971
 972         if (!priv->ppio)
 973                 return;
 974
 975         if (priv->isolated)
 976                 return;
 977
 978         ret = pp2_ppio_remove_mac_addr(priv->ppio,
 979                                        dev->data->mac_addrs[index].addr_bytes);
 980         if (ret) {
 981                 ether_format_addr(buf, sizeof(buf),
 982                                   &dev->data->mac_addrs[index]);
 983                 MRVL_LOG(ERR, "Failed to remove mac %s", buf);
 984         }
 985 }
 986
 987 /**
 988  * DPDK callback to add a MAC address.
 989  *
 990  * @param dev
 991  *   Pointer to Ethernet device structure.
 992  * @param mac_addr
 993  *   MAC address to register.
 994  * @param index
 995  *   MAC address index.
 996  * @param vmdq
 997  *   VMDq pool index to associate address with (unused).
 998  *
 999  * @return
1000  *   0 on success, negative error value otherwise.
1001  */
1002 static int
1003 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
1004                   uint32_t index, uint32_t vmdq __rte_unused)
1005 {
1006         struct mrvl_priv *priv = dev->data->dev_private;
1007         char buf[ETHER_ADDR_FMT_SIZE];
1008         int ret;
1009
1010         if (priv->isolated)
1011                 return -ENOTSUP;
1012
1013         if (index == 0)
1014                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
1015                 return -1;
1016
1017         if (!priv->ppio)
1018                 return 0;
1019
1020         /*
1021          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
1022          * parameter uc_filter_max. Maximum number of mc addresses is then
1023          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
1024          * 21 respectively.
1025          *
1026          * If more than uc_filter_max uc addresses were added to filter list
1027          * then NIC will switch to promiscuous mode automatically.
1028          *
1029          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
1030          * were added to filter list then NIC will switch to all-multicast mode
1031          * automatically.
1032          */
1033         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
1034         if (ret) {
1035                 ether_format_addr(buf, sizeof(buf), mac_addr);
1036                 MRVL_LOG(ERR, "Failed to add mac %s", buf);
1037                 return -1;
1038         }
1039
1040         return 0;
1041 }
1042
1043 /**
1044  * DPDK callback to set the primary MAC address.
1045  *
1046  * @param dev
1047  *   Pointer to Ethernet device structure.
1048  * @param mac_addr
1049  *   MAC address to register.
1050  *
1051  * @return
1052  *   0 on success, negative error value otherwise.
1053  */
1054 static int
1055 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1056 {
1057         struct mrvl_priv *priv = dev->data->dev_private;
1058         int ret;
1059
1060         if (!priv->ppio)
1061                 return 0;
1062
1063         if (priv->isolated)
1064                 return -ENOTSUP;
1065
1066         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
1067         if (ret) {
1068                 char buf[ETHER_ADDR_FMT_SIZE];
1069                 ether_format_addr(buf, sizeof(buf), mac_addr);
1070                 MRVL_LOG(ERR, "Failed to set mac to %s", buf);
1071         }
1072
1073         return ret;
1074 }
1075
1076 /**
1077  * DPDK callback to get device statistics.
1078  *
1079  * @param dev
1080  *   Pointer to Ethernet device structure.
1081  * @param stats
1082  *   Stats structure output buffer.
1083  *
1084  * @return
1085  *   0 on success, negative error value otherwise.
1086  */
1087 static int
1088 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1089 {
1090         struct mrvl_priv *priv = dev->data->dev_private;
1091         struct pp2_ppio_statistics ppio_stats;
1092         uint64_t drop_mac = 0;
1093         unsigned int i, idx, ret;
1094
1095         if (!priv->ppio)
1096                 return -EPERM;
1097
1098         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1099                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1100                 struct pp2_ppio_inq_statistics rx_stats;
1101
1102                 if (!rxq)
1103                         continue;
1104
1105                 idx = rxq->queue_id;
1106                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1107                         MRVL_LOG(ERR,
1108                                 "rx queue %d stats out of range (0 - %d)",
1109                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1110                         continue;
1111                 }
1112
1113                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
1114                                                   priv->rxq_map[idx].tc,
1115                                                   priv->rxq_map[idx].inq,
1116                                                   &rx_stats, 0);
1117                 if (unlikely(ret)) {
1118                         MRVL_LOG(ERR,
1119                                 "Failed to update rx queue %d stats", idx);
1120                         break;
1121                 }
1122
1123                 stats->q_ibytes[idx] = rxq->bytes_recv;
1124                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
1125                 stats->q_errors[idx] = rx_stats.drop_early +
1126                                        rx_stats.drop_fullq +
1127                                        rx_stats.drop_bm +
1128                                        rxq->drop_mac;
1129                 stats->ibytes += rxq->bytes_recv;
1130                 drop_mac += rxq->drop_mac;
1131         }
1132
1133         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1134                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1135                 struct pp2_ppio_outq_statistics tx_stats;
1136
1137                 if (!txq)
1138                         continue;
1139
1140                 idx = txq->queue_id;
1141                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1142                         MRVL_LOG(ERR,
1143                                 "tx queue %d stats out of range (0 - %d)",
1144                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1145                 }
1146
1147                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1148                                                    &tx_stats, 0);
1149                 if (unlikely(ret)) {
1150                         MRVL_LOG(ERR,
1151                                 "Failed to update tx queue %d stats", idx);
1152                         break;
1153                 }
1154
1155                 stats->q_opackets[idx] = tx_stats.deq_desc;
1156                 stats->q_obytes[idx] = txq->bytes_sent;
1157                 stats->obytes += txq->bytes_sent;
1158         }
1159
1160         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1161         if (unlikely(ret)) {
1162                 MRVL_LOG(ERR, "Failed to update port statistics");
1163                 return ret;
1164         }
1165
1166         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1167         stats->opackets += ppio_stats.tx_packets;
1168         stats->imissed += ppio_stats.rx_fullq_dropped +
1169                           ppio_stats.rx_bm_dropped +
1170                           ppio_stats.rx_early_dropped +
1171                           ppio_stats.rx_fifo_dropped +
1172                           ppio_stats.rx_cls_dropped;
1173         stats->ierrors = drop_mac;
1174
1175         return 0;
1176 }
1177
1178 /**
1179  * DPDK callback to clear device statistics.
1180  *
1181  * @param dev
1182  *   Pointer to Ethernet device structure.
1183  */
1184 static void
1185 mrvl_stats_reset(struct rte_eth_dev *dev)
1186 {
1187         struct mrvl_priv *priv = dev->data->dev_private;
1188         int i;
1189
1190         if (!priv->ppio)
1191                 return;
1192
1193         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1194                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1195
1196                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1197                                             priv->rxq_map[i].inq, NULL, 1);
1198                 rxq->bytes_recv = 0;
1199                 rxq->drop_mac = 0;
1200         }
1201
1202         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1203                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1204
1205                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1206                 txq->bytes_sent = 0;
1207         }
1208
1209         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1210 }
1211
1212 /**
1213  * DPDK callback to get extended statistics.
1214  *
1215  * @param dev
1216  *   Pointer to Ethernet device structure.
1217  * @param stats
1218  *   Pointer to xstats table.
1219  * @param n
1220  *   Number of entries in xstats table.
1221  * @return
1222  *   Negative value on error, number of read xstats otherwise.
1223  */
1224 static int
1225 mrvl_xstats_get(struct rte_eth_dev *dev,
1226                 struct rte_eth_xstat *stats, unsigned int n)
1227 {
1228         struct mrvl_priv *priv = dev->data->dev_private;
1229         struct pp2_ppio_statistics ppio_stats;
1230         unsigned int i;
1231
1232         if (!stats)
1233                 return 0;
1234
1235         pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1236         for (i = 0; i < n && i < RTE_DIM(mrvl_xstats_tbl); i++) {
1237                 uint64_t val;
1238
1239                 if (mrvl_xstats_tbl[i].size == sizeof(uint32_t))
1240                         val = *(uint32_t *)((uint8_t *)&ppio_stats +
1241                                             mrvl_xstats_tbl[i].offset);
1242                 else if (mrvl_xstats_tbl[i].size == sizeof(uint64_t))
1243                         val = *(uint64_t *)((uint8_t *)&ppio_stats +
1244                                             mrvl_xstats_tbl[i].offset);
1245                 else
1246                         return -EINVAL;
1247
1248                 stats[i].id = i;
1249                 stats[i].value = val;
1250         }
1251
1252         return n;
1253 }
1254
1255 /**
1256  * DPDK callback to reset extended statistics.
1257  *
1258  * @param dev
1259  *   Pointer to Ethernet device structure.
1260  */
1261 static void
1262 mrvl_xstats_reset(struct rte_eth_dev *dev)
1263 {
1264         mrvl_stats_reset(dev);
1265 }
1266
1267 /**
1268  * DPDK callback to get extended statistics names.
1269  *
1270  * @param dev (unused)
1271  *   Pointer to Ethernet device structure.
1272  * @param xstats_names
1273  *   Pointer to xstats names table.
1274  * @param size
1275  *   Size of the xstats names table.
1276  * @return
1277  *   Number of read names.
1278  */
1279 static int
1280 mrvl_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
1281                       struct rte_eth_xstat_name *xstats_names,
1282                       unsigned int size)
1283 {
1284         unsigned int i;
1285
1286         if (!xstats_names)
1287                 return RTE_DIM(mrvl_xstats_tbl);
1288
1289         for (i = 0; i < size && i < RTE_DIM(mrvl_xstats_tbl); i++)
1290                 snprintf(xstats_names[i].name, RTE_ETH_XSTATS_NAME_SIZE, "%s",
1291                          mrvl_xstats_tbl[i].name);
1292
1293         return size;
1294 }
1295
1296 /**
1297  * DPDK callback to get information about the device.
1298  *
1299  * @param dev
1300  *   Pointer to Ethernet device structure (unused).
1301  * @param info
1302  *   Info structure output buffer.
1303  */
1304 static void
1305 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
1306                    struct rte_eth_dev_info *info)
1307 {
1308         info->speed_capa = ETH_LINK_SPEED_10M |
1309                            ETH_LINK_SPEED_100M |
1310                            ETH_LINK_SPEED_1G |
1311                            ETH_LINK_SPEED_10G;
1312
1313         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1314         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1315         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1316
1317         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1318         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1319         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1320
1321         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1322         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1323         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1324
1325         info->rx_offload_capa = MRVL_RX_OFFLOADS;
1326         info->rx_queue_offload_capa = MRVL_RX_OFFLOADS;
1327
1328         info->tx_offload_capa = MRVL_TX_OFFLOADS;
1329         info->tx_queue_offload_capa = MRVL_TX_OFFLOADS;
1330
1331         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1332                                        ETH_RSS_NONFRAG_IPV4_TCP |
1333                                        ETH_RSS_NONFRAG_IPV4_UDP;
1334
1335         /* By default packets are dropped if no descriptors are available */
1336         info->default_rxconf.rx_drop_en = 1;
1337         info->default_rxconf.offloads = DEV_RX_OFFLOAD_CRC_STRIP;
1338
1339         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1340 }
1341
1342 /**
1343  * Return supported packet types.
1344  *
1345  * @param dev
1346  *   Pointer to Ethernet device structure (unused).
1347  *
1348  * @return
1349  *   Const pointer to the table with supported packet types.
1350  */
1351 static const uint32_t *
1352 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1353 {
1354         static const uint32_t ptypes[] = {
1355                 RTE_PTYPE_L2_ETHER,
1356                 RTE_PTYPE_L2_ETHER_VLAN,
1357                 RTE_PTYPE_L2_ETHER_QINQ,
1358                 RTE_PTYPE_L3_IPV4,
1359                 RTE_PTYPE_L3_IPV4_EXT,
1360                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1361                 RTE_PTYPE_L3_IPV6,
1362                 RTE_PTYPE_L3_IPV6_EXT,
1363                 RTE_PTYPE_L2_ETHER_ARP,
1364                 RTE_PTYPE_L4_TCP,
1365                 RTE_PTYPE_L4_UDP
1366         };
1367
1368         return ptypes;
1369 }
1370
1371 /**
1372  * DPDK callback to get information about specific receive queue.
1373  *
1374  * @param dev
1375  *   Pointer to Ethernet device structure.
1376  * @param rx_queue_id
1377  *   Receive queue index.
1378  * @param qinfo
1379  *   Receive queue information structure.
1380  */
1381 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1382                               struct rte_eth_rxq_info *qinfo)
1383 {
1384         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1385         struct mrvl_priv *priv = dev->data->dev_private;
1386         int inq = priv->rxq_map[rx_queue_id].inq;
1387         int tc = priv->rxq_map[rx_queue_id].tc;
1388         struct pp2_ppio_tc_params *tc_params =
1389                 &priv->ppio_params.inqs_params.tcs_params[tc];
1390
1391         qinfo->mp = q->mp;
1392         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1393 }
1394
1395 /**
1396  * DPDK callback to get information about specific transmit queue.
1397  *
1398  * @param dev
1399  *   Pointer to Ethernet device structure.
1400  * @param tx_queue_id
1401  *   Transmit queue index.
1402  * @param qinfo
1403  *   Transmit queue information structure.
1404  */
1405 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1406                               struct rte_eth_txq_info *qinfo)
1407 {
1408         struct mrvl_priv *priv = dev->data->dev_private;
1409         struct mrvl_txq *txq = dev->data->tx_queues[tx_queue_id];
1410
1411         qinfo->nb_desc =
1412                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1413         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1414 }
1415
1416 /**
1417  * DPDK callback to Configure a VLAN filter.
1418  *
1419  * @param dev
1420  *   Pointer to Ethernet device structure.
1421  * @param vlan_id
1422  *   VLAN ID to filter.
1423  * @param on
1424  *   Toggle filter.
1425  *
1426  * @return
1427  *   0 on success, negative error value otherwise.
1428  */
1429 static int
1430 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1431 {
1432         struct mrvl_priv *priv = dev->data->dev_private;
1433
1434         if (!priv->ppio)
1435                 return -EPERM;
1436
1437         if (priv->isolated)
1438                 return -ENOTSUP;
1439
1440         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1441                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1442 }
1443
1444 /**
1445  * Release buffers to hardware bpool (buffer-pool)
1446  *
1447  * @param rxq
1448  *   Receive queue pointer.
1449  * @param num
1450  *   Number of buffers to release to bpool.
1451  *
1452  * @return
1453  *   0 on success, negative error value otherwise.
1454  */
1455 static int
1456 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1457 {
1458         struct buff_release_entry entries[MRVL_PP2_RXD_MAX];
1459         struct rte_mbuf *mbufs[MRVL_PP2_RXD_MAX];
1460         int i, ret;
1461         unsigned int core_id;
1462         struct pp2_hif *hif;
1463         struct pp2_bpool *bpool;
1464
1465         core_id = rte_lcore_id();
1466         if (core_id == LCORE_ID_ANY)
1467                 core_id = 0;
1468
1469         hif = mrvl_get_hif(rxq->priv, core_id);
1470         if (!hif)
1471                 return -1;
1472
1473         bpool = rxq->priv->bpool;
1474
1475         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1476         if (ret)
1477                 return ret;
1478
1479         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1480                 cookie_addr_high =
1481                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1482
1483         for (i = 0; i < num; i++) {
1484                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1485                         != cookie_addr_high) {
1486                         MRVL_LOG(ERR,
1487                                 "mbuf virtual addr high 0x%lx out of range",
1488                                 (uint64_t)mbufs[i] >> 32);
1489                         goto out;
1490                 }
1491
1492                 entries[i].buff.addr =
1493                         rte_mbuf_data_iova_default(mbufs[i]);
1494                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
1495                 entries[i].bpool = bpool;
1496         }
1497
1498         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1499         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1500
1501         if (i != num)
1502                 goto out;
1503
1504         return 0;
1505 out:
1506         for (; i < num; i++)
1507                 rte_pktmbuf_free(mbufs[i]);
1508
1509         return -1;
1510 }
1511
1512 /**
1513  * DPDK callback to configure the receive queue.
1514  *
1515  * @param dev
1516  *   Pointer to Ethernet device structure.
1517  * @param idx
1518  *   RX queue index.
1519  * @param desc
1520  *   Number of descriptors to configure in queue.
1521  * @param socket
1522  *   NUMA socket on which memory must be allocated.
1523  * @param conf
1524  *   Thresholds parameters.
1525  * @param mp
1526  *   Memory pool for buffer allocations.
1527  *
1528  * @return
1529  *   0 on success, negative error value otherwise.
1530  */
1531 static int
1532 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1533                     unsigned int socket,
1534                     const struct rte_eth_rxconf *conf,
1535                     struct rte_mempool *mp)
1536 {
1537         struct mrvl_priv *priv = dev->data->dev_private;
1538         struct mrvl_rxq *rxq;
1539         uint32_t min_size,
1540                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1541         int ret, tc, inq;
1542         uint64_t offloads;
1543
1544         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1545
1546         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1547                 /*
1548                  * Unknown TC mapping, mapping will not have a correct queue.
1549                  */
1550                 MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu",
1551                         idx, priv->ppio_id);
1552                 return -EFAULT;
1553         }
1554
1555         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
1556                    MRVL_PKT_EFFEC_OFFS;
1557         if (min_size < max_rx_pkt_len) {
1558                 MRVL_LOG(ERR,
1559                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.",
1560                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
1561                         MRVL_PKT_EFFEC_OFFS,
1562                         max_rx_pkt_len);
1563                 return -EINVAL;
1564         }
1565
1566         if (dev->data->rx_queues[idx]) {
1567                 rte_free(dev->data->rx_queues[idx]);
1568                 dev->data->rx_queues[idx] = NULL;
1569         }
1570
1571         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1572         if (!rxq)
1573                 return -ENOMEM;
1574
1575         rxq->priv = priv;
1576         rxq->mp = mp;
1577         rxq->cksum_enabled = offloads & DEV_RX_OFFLOAD_IPV4_CKSUM;
1578         rxq->queue_id = idx;
1579         rxq->port_id = dev->data->port_id;
1580         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1581
1582         tc = priv->rxq_map[rxq->queue_id].tc,
1583         inq = priv->rxq_map[rxq->queue_id].inq;
1584         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1585                 desc;
1586
1587         ret = mrvl_fill_bpool(rxq, desc);
1588         if (ret) {
1589                 rte_free(rxq);
1590                 return ret;
1591         }
1592
1593         priv->bpool_init_size += desc;
1594
1595         dev->data->rx_queues[idx] = rxq;
1596
1597         return 0;
1598 }
1599
1600 /**
1601  * DPDK callback to release the receive queue.
1602  *
1603  * @param rxq
1604  *   Generic receive queue pointer.
1605  */
1606 static void
1607 mrvl_rx_queue_release(void *rxq)
1608 {
1609         struct mrvl_rxq *q = rxq;
1610         struct pp2_ppio_tc_params *tc_params;
1611         int i, num, tc, inq;
1612         struct pp2_hif *hif;
1613         unsigned int core_id = rte_lcore_id();
1614
1615         if (core_id == LCORE_ID_ANY)
1616                 core_id = 0;
1617
1618         if (!q)
1619                 return;
1620
1621         hif = mrvl_get_hif(q->priv, core_id);
1622
1623         if (!hif)
1624                 return;
1625
1626         tc = q->priv->rxq_map[q->queue_id].tc;
1627         inq = q->priv->rxq_map[q->queue_id].inq;
1628         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1629         num = tc_params->inqs_params[inq].size;
1630         for (i = 0; i < num; i++) {
1631                 struct pp2_buff_inf inf;
1632                 uint64_t addr;
1633
1634                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
1635                 addr = cookie_addr_high | inf.cookie;
1636                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1637         }
1638
1639         rte_free(q);
1640 }
1641
1642 /**
1643  * DPDK callback to configure the transmit queue.
1644  *
1645  * @param dev
1646  *   Pointer to Ethernet device structure.
1647  * @param idx
1648  *   Transmit queue index.
1649  * @param desc
1650  *   Number of descriptors to configure in the queue.
1651  * @param socket
1652  *   NUMA socket on which memory must be allocated.
1653  * @param conf
1654  *   Tx queue configuration parameters.
1655  *
1656  * @return
1657  *   0 on success, negative error value otherwise.
1658  */
1659 static int
1660 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1661                     unsigned int socket,
1662                     const struct rte_eth_txconf *conf)
1663 {
1664         struct mrvl_priv *priv = dev->data->dev_private;
1665         struct mrvl_txq *txq;
1666
1667         if (dev->data->tx_queues[idx]) {
1668                 rte_free(dev->data->tx_queues[idx]);
1669                 dev->data->tx_queues[idx] = NULL;
1670         }
1671
1672         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1673         if (!txq)
1674                 return -ENOMEM;
1675
1676         txq->priv = priv;
1677         txq->queue_id = idx;
1678         txq->port_id = dev->data->port_id;
1679         txq->tx_deferred_start = conf->tx_deferred_start;
1680         dev->data->tx_queues[idx] = txq;
1681
1682         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1683
1684         return 0;
1685 }
1686
1687 /**
1688  * DPDK callback to release the transmit queue.
1689  *
1690  * @param txq
1691  *   Generic transmit queue pointer.
1692  */
1693 static void
1694 mrvl_tx_queue_release(void *txq)
1695 {
1696         struct mrvl_txq *q = txq;
1697
1698         if (!q)
1699                 return;
1700
1701         rte_free(q);
1702 }
1703
1704 /**
1705  * DPDK callback to get flow control configuration.
1706  *
1707  * @param dev
1708  *  Pointer to Ethernet device structure.
1709  * @param fc_conf
1710  *  Pointer to the flow control configuration.
1711  *
1712  * @return
1713  *  0 on success, negative error value otherwise.
1714  */
1715 static int
1716 mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1717 {
1718         struct mrvl_priv *priv = dev->data->dev_private;
1719         int ret, en;
1720
1721         if (!priv)
1722                 return -EPERM;
1723
1724         ret = pp2_ppio_get_rx_pause(priv->ppio, &en);
1725         if (ret) {
1726                 MRVL_LOG(ERR, "Failed to read rx pause state");
1727                 return ret;
1728         }
1729
1730         fc_conf->mode = en ? RTE_FC_RX_PAUSE : RTE_FC_NONE;
1731
1732         return 0;
1733 }
1734
1735 /**
1736  * DPDK callback to set flow control configuration.
1737  *
1738  * @param dev
1739  *  Pointer to Ethernet device structure.
1740  * @param fc_conf
1741  *  Pointer to the flow control configuration.
1742  *
1743  * @return
1744  *  0 on success, negative error value otherwise.
1745  */
1746 static int
1747 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1748 {
1749         struct mrvl_priv *priv = dev->data->dev_private;
1750
1751         if (!priv)
1752                 return -EPERM;
1753
1754         if (fc_conf->high_water ||
1755             fc_conf->low_water ||
1756             fc_conf->pause_time ||
1757             fc_conf->mac_ctrl_frame_fwd ||
1758             fc_conf->autoneg) {
1759                 MRVL_LOG(ERR, "Flowctrl parameter is not supported");
1760
1761                 return -EINVAL;
1762         }
1763
1764         if (fc_conf->mode == RTE_FC_NONE ||
1765             fc_conf->mode == RTE_FC_RX_PAUSE) {
1766                 int ret, en;
1767
1768                 en = fc_conf->mode == RTE_FC_NONE ? 0 : 1;
1769                 ret = pp2_ppio_set_rx_pause(priv->ppio, en);
1770                 if (ret)
1771                         MRVL_LOG(ERR,
1772                                 "Failed to change flowctrl on RX side");
1773
1774                 return ret;
1775         }
1776
1777         return 0;
1778 }
1779
1780 /**
1781  * Update RSS hash configuration
1782  *
1783  * @param dev
1784  *   Pointer to Ethernet device structure.
1785  * @param rss_conf
1786  *   Pointer to RSS configuration.
1787  *
1788  * @return
1789  *   0 on success, negative error value otherwise.
1790  */
1791 static int
1792 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1793                      struct rte_eth_rss_conf *rss_conf)
1794 {
1795         struct mrvl_priv *priv = dev->data->dev_private;
1796
1797         if (priv->isolated)
1798                 return -ENOTSUP;
1799
1800         return mrvl_configure_rss(priv, rss_conf);
1801 }
1802
1803 /**
1804  * DPDK callback to get RSS hash configuration.
1805  *
1806  * @param dev
1807  *   Pointer to Ethernet device structure.
1808  * @rss_conf
1809  *   Pointer to RSS configuration.
1810  *
1811  * @return
1812  *   Always 0.
1813  */
1814 static int
1815 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1816                        struct rte_eth_rss_conf *rss_conf)
1817 {
1818         struct mrvl_priv *priv = dev->data->dev_private;
1819         enum pp2_ppio_hash_type hash_type =
1820                 priv->ppio_params.inqs_params.hash_type;
1821
1822         rss_conf->rss_key = NULL;
1823
1824         if (hash_type == PP2_PPIO_HASH_T_NONE)
1825                 rss_conf->rss_hf = 0;
1826         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1827                 rss_conf->rss_hf = ETH_RSS_IPV4;
1828         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1829                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1830         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1831                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1832
1833         return 0;
1834 }
1835
1836 /**
1837  * DPDK callback to get rte_flow callbacks.
1838  *
1839  * @param dev
1840  *   Pointer to the device structure.
1841  * @param filer_type
1842  *   Flow filter type.
1843  * @param filter_op
1844  *   Flow filter operation.
1845  * @param arg
1846  *   Pointer to pass the flow ops.
1847  *
1848  * @return
1849  *   0 on success, negative error value otherwise.
1850  */
1851 static int
1852 mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
1853                      enum rte_filter_type filter_type,
1854                      enum rte_filter_op filter_op, void *arg)
1855 {
1856         switch (filter_type) {
1857         case RTE_ETH_FILTER_GENERIC:
1858                 if (filter_op != RTE_ETH_FILTER_GET)
1859                         return -EINVAL;
1860                 *(const void **)arg = &mrvl_flow_ops;
1861                 return 0;
1862         default:
1863                 MRVL_LOG(WARNING, "Filter type (%d) not supported",
1864                                 filter_type);
1865                 return -EINVAL;
1866         }
1867 }
1868
1869 static const struct eth_dev_ops mrvl_ops = {
1870         .dev_configure = mrvl_dev_configure,
1871         .dev_start = mrvl_dev_start,
1872         .dev_stop = mrvl_dev_stop,
1873         .dev_set_link_up = mrvl_dev_set_link_up,
1874         .dev_set_link_down = mrvl_dev_set_link_down,
1875         .dev_close = mrvl_dev_close,
1876         .link_update = mrvl_link_update,
1877         .promiscuous_enable = mrvl_promiscuous_enable,
1878         .allmulticast_enable = mrvl_allmulticast_enable,
1879         .promiscuous_disable = mrvl_promiscuous_disable,
1880         .allmulticast_disable = mrvl_allmulticast_disable,
1881         .mac_addr_remove = mrvl_mac_addr_remove,
1882         .mac_addr_add = mrvl_mac_addr_add,
1883         .mac_addr_set = mrvl_mac_addr_set,
1884         .mtu_set = mrvl_mtu_set,
1885         .stats_get = mrvl_stats_get,
1886         .stats_reset = mrvl_stats_reset,
1887         .xstats_get = mrvl_xstats_get,
1888         .xstats_reset = mrvl_xstats_reset,
1889         .xstats_get_names = mrvl_xstats_get_names,
1890         .dev_infos_get = mrvl_dev_infos_get,
1891         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
1892         .rxq_info_get = mrvl_rxq_info_get,
1893         .txq_info_get = mrvl_txq_info_get,
1894         .vlan_filter_set = mrvl_vlan_filter_set,
1895         .tx_queue_start = mrvl_tx_queue_start,
1896         .tx_queue_stop = mrvl_tx_queue_stop,
1897         .rx_queue_setup = mrvl_rx_queue_setup,
1898         .rx_queue_release = mrvl_rx_queue_release,
1899         .tx_queue_setup = mrvl_tx_queue_setup,
1900         .tx_queue_release = mrvl_tx_queue_release,
1901         .flow_ctrl_get = mrvl_flow_ctrl_get,
1902         .flow_ctrl_set = mrvl_flow_ctrl_set,
1903         .rss_hash_update = mrvl_rss_hash_update,
1904         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1905         .filter_ctrl = mrvl_eth_filter_ctrl,
1906 };
1907
1908 /**
1909  * Return packet type information and l3/l4 offsets.
1910  *
1911  * @param desc
1912  *   Pointer to the received packet descriptor.
1913  * @param l3_offset
1914  *   l3 packet offset.
1915  * @param l4_offset
1916  *   l4 packet offset.
1917  *
1918  * @return
1919  *   Packet type information.
1920  */
1921 static inline uint64_t
1922 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
1923                                     uint8_t *l3_offset, uint8_t *l4_offset)
1924 {
1925         enum pp2_inq_l3_type l3_type;
1926         enum pp2_inq_l4_type l4_type;
1927         enum pp2_inq_vlan_tag vlan_tag;
1928         uint64_t packet_type;
1929
1930         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
1931         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
1932         pp2_ppio_inq_desc_get_vlan_tag(desc, &vlan_tag);
1933
1934         packet_type = RTE_PTYPE_L2_ETHER;
1935
1936         switch (vlan_tag) {
1937         case PP2_INQ_VLAN_TAG_SINGLE:
1938                 packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
1939                 break;
1940         case PP2_INQ_VLAN_TAG_DOUBLE:
1941         case PP2_INQ_VLAN_TAG_TRIPLE:
1942                 packet_type |= RTE_PTYPE_L2_ETHER_QINQ;
1943                 break;
1944         default:
1945                 break;
1946         }
1947
1948         switch (l3_type) {
1949         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
1950                 packet_type |= RTE_PTYPE_L3_IPV4;
1951                 break;
1952         case PP2_INQ_L3_TYPE_IPV4_OK:
1953                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
1954                 break;
1955         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
1956                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
1957                 break;
1958         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
1959                 packet_type |= RTE_PTYPE_L3_IPV6;
1960                 break;
1961         case PP2_INQ_L3_TYPE_IPV6_EXT:
1962                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
1963                 break;
1964         case PP2_INQ_L3_TYPE_ARP:
1965                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
1966                 /*
1967                  * In case of ARP l4_offset is set to wrong value.
1968                  * Set it to proper one so that later on mbuf->l3_len can be
1969                  * calculated subtracting l4_offset and l3_offset.
1970                  */
1971                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
1972                 break;
1973         default:
1974                 MRVL_LOG(DEBUG, "Failed to recognise l3 packet type");
1975                 break;
1976         }
1977
1978         switch (l4_type) {
1979         case PP2_INQ_L4_TYPE_TCP:
1980                 packet_type |= RTE_PTYPE_L4_TCP;
1981                 break;
1982         case PP2_INQ_L4_TYPE_UDP:
1983                 packet_type |= RTE_PTYPE_L4_UDP;
1984                 break;
1985         default:
1986                 MRVL_LOG(DEBUG, "Failed to recognise l4 packet type");
1987                 break;
1988         }
1989
1990         return packet_type;
1991 }
1992
1993 /**
1994  * Get offload information from the received packet descriptor.
1995  *
1996  * @param desc
1997  *   Pointer to the received packet descriptor.
1998  *
1999  * @return
2000  *   Mbuf offload flags.
2001  */
2002 static inline uint64_t
2003 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
2004 {
2005         uint64_t flags;
2006         enum pp2_inq_desc_status status;
2007
2008         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
2009         if (unlikely(status != PP2_DESC_ERR_OK))
2010                 flags = PKT_RX_IP_CKSUM_BAD;
2011         else
2012                 flags = PKT_RX_IP_CKSUM_GOOD;
2013
2014         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
2015         if (unlikely(status != PP2_DESC_ERR_OK))
2016                 flags |= PKT_RX_L4_CKSUM_BAD;
2017         else
2018                 flags |= PKT_RX_L4_CKSUM_GOOD;
2019
2020         return flags;
2021 }
2022
2023 /**
2024  * DPDK callback for receive.
2025  *
2026  * @param rxq
2027  *   Generic pointer to the receive queue.
2028  * @param rx_pkts
2029  *   Array to store received packets.
2030  * @param nb_pkts
2031  *   Maximum number of packets in array.
2032  *
2033  * @return
2034  *   Number of packets successfully received.
2035  */
2036 static uint16_t
2037 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2038 {
2039         struct mrvl_rxq *q = rxq;
2040         struct pp2_ppio_desc descs[nb_pkts];
2041         struct pp2_bpool *bpool;
2042         int i, ret, rx_done = 0;
2043         int num;
2044         struct pp2_hif *hif;
2045         unsigned int core_id = rte_lcore_id();
2046
2047         hif = mrvl_get_hif(q->priv, core_id);
2048
2049         if (unlikely(!q->priv->ppio || !hif))
2050                 return 0;
2051
2052         bpool = q->priv->bpool;
2053
2054         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
2055                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
2056         if (unlikely(ret < 0)) {
2057                 MRVL_LOG(ERR, "Failed to receive packets");
2058                 return 0;
2059         }
2060         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
2061
2062         for (i = 0; i < nb_pkts; i++) {
2063                 struct rte_mbuf *mbuf;
2064                 uint8_t l3_offset, l4_offset;
2065                 enum pp2_inq_desc_status status;
2066                 uint64_t addr;
2067
2068                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2069                         struct pp2_ppio_desc *pref_desc;
2070                         u64 pref_addr;
2071
2072                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
2073                         pref_addr = cookie_addr_high |
2074                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
2075                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
2076                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
2077                 }
2078
2079                 addr = cookie_addr_high |
2080                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
2081                 mbuf = (struct rte_mbuf *)addr;
2082                 rte_pktmbuf_reset(mbuf);
2083
2084                 /* drop packet in case of mac, overrun or resource error */
2085                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
2086                 if (unlikely(status != PP2_DESC_ERR_OK)) {
2087                         struct pp2_buff_inf binf = {
2088                                 .addr = rte_mbuf_data_iova_default(mbuf),
2089                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
2090                         };
2091
2092                         pp2_bpool_put_buff(hif, bpool, &binf);
2093                         mrvl_port_bpool_size
2094                                 [bpool->pp2_id][bpool->id][core_id]++;
2095                         q->drop_mac++;
2096                         continue;
2097                 }
2098
2099                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
2100                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
2101                 mbuf->data_len = mbuf->pkt_len;
2102                 mbuf->port = q->port_id;
2103                 mbuf->packet_type =
2104                         mrvl_desc_to_packet_type_and_offset(&descs[i],
2105                                                             &l3_offset,
2106                                                             &l4_offset);
2107                 mbuf->l2_len = l3_offset;
2108                 mbuf->l3_len = l4_offset - l3_offset;
2109
2110                 if (likely(q->cksum_enabled))
2111                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
2112
2113                 rx_pkts[rx_done++] = mbuf;
2114                 q->bytes_recv += mbuf->pkt_len;
2115         }
2116
2117         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
2118                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
2119
2120                 if (unlikely(num <= q->priv->bpool_min_size ||
2121                              (!rx_done && num < q->priv->bpool_init_size))) {
2122                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
2123                         if (ret)
2124                                 MRVL_LOG(ERR, "Failed to fill bpool");
2125                 } else if (unlikely(num > q->priv->bpool_max_size)) {
2126                         int i;
2127                         int pkt_to_remove = num - q->priv->bpool_init_size;
2128                         struct rte_mbuf *mbuf;
2129                         struct pp2_buff_inf buff;
2130
2131                         MRVL_LOG(DEBUG,
2132                                 "port-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)",
2133                                 bpool->pp2_id, q->priv->ppio->port_id,
2134                                 bpool->id, pkt_to_remove, num,
2135                                 q->priv->bpool_init_size);
2136
2137                         for (i = 0; i < pkt_to_remove; i++) {
2138                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
2139                                 if (ret)
2140                                         break;
2141                                 mbuf = (struct rte_mbuf *)
2142                                         (cookie_addr_high | buff.cookie);
2143                                 rte_pktmbuf_free(mbuf);
2144                         }
2145                         mrvl_port_bpool_size
2146                                 [bpool->pp2_id][bpool->id][core_id] -= i;
2147                 }
2148                 rte_spinlock_unlock(&q->priv->lock);
2149         }
2150
2151         return rx_done;
2152 }
2153
2154 /**
2155  * Prepare offload information.
2156  *
2157  * @param ol_flags
2158  *   Offload flags.
2159  * @param packet_type
2160  *   Packet type bitfield.
2161  * @param l3_type
2162  *   Pointer to the pp2_ouq_l3_type structure.
2163  * @param l4_type
2164  *   Pointer to the pp2_outq_l4_type structure.
2165  * @param gen_l3_cksum
2166  *   Will be set to 1 in case l3 checksum is computed.
2167  * @param l4_cksum
2168  *   Will be set to 1 in case l4 checksum is computed.
2169  *
2170  * @return
2171  *   0 on success, negative error value otherwise.
2172  */
2173 static inline int
2174 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
2175                         enum pp2_outq_l3_type *l3_type,
2176                         enum pp2_outq_l4_type *l4_type,
2177                         int *gen_l3_cksum,
2178                         int *gen_l4_cksum)
2179 {
2180         /*
2181          * Based on ol_flags prepare information
2182          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
2183          * for offloading.
2184          */
2185         if (ol_flags & PKT_TX_IPV4) {
2186                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
2187                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
2188         } else if (ol_flags & PKT_TX_IPV6) {
2189                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
2190                 /* no checksum for ipv6 header */
2191                 *gen_l3_cksum = 0;
2192         } else {
2193                 /* if something different then stop processing */
2194                 return -1;
2195         }
2196
2197         ol_flags &= PKT_TX_L4_MASK;
2198         if ((packet_type & RTE_PTYPE_L4_TCP) &&
2199             ol_flags == PKT_TX_TCP_CKSUM) {
2200                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
2201                 *gen_l4_cksum = 1;
2202         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
2203                    ol_flags == PKT_TX_UDP_CKSUM) {
2204                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
2205                 *gen_l4_cksum = 1;
2206         } else {
2207                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
2208                 /* no checksum for other type */
2209                 *gen_l4_cksum = 0;
2210         }
2211
2212         return 0;
2213 }
2214
2215 /**
2216  * Release already sent buffers to bpool (buffer-pool).
2217  *
2218  * @param ppio
2219  *   Pointer to the port structure.
2220  * @param hif
2221  *   Pointer to the MUSDK hardware interface.
2222  * @param sq
2223  *   Pointer to the shadow queue.
2224  * @param qid
2225  *   Queue id number.
2226  * @param force
2227  *   Force releasing packets.
2228  */
2229 static inline void
2230 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
2231                        unsigned int core_id, struct mrvl_shadow_txq *sq,
2232                        int qid, int force)
2233 {
2234         struct buff_release_entry *entry;
2235         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
2236         int i;
2237
2238         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
2239
2240         sq->num_to_release += nb_done;
2241
2242         if (likely(!force &&
2243                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
2244                 return;
2245
2246         nb_done = sq->num_to_release;
2247         sq->num_to_release = 0;
2248
2249         for (i = 0; i < nb_done; i++) {
2250                 entry = &sq->ent[sq->tail + num];
2251                 if (unlikely(!entry->buff.addr)) {
2252                         MRVL_LOG(ERR,
2253                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!",
2254                                 sq->tail, (u64)entry->buff.cookie,
2255                                 (u64)entry->buff.addr);
2256                         skip_bufs = 1;
2257                         goto skip;
2258                 }
2259
2260                 if (unlikely(!entry->bpool)) {
2261                         struct rte_mbuf *mbuf;
2262
2263                         mbuf = (struct rte_mbuf *)
2264                                (cookie_addr_high | entry->buff.cookie);
2265                         rte_pktmbuf_free(mbuf);
2266                         skip_bufs = 1;
2267                         goto skip;
2268                 }
2269
2270                 mrvl_port_bpool_size
2271                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
2272                 num++;
2273                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
2274                         goto skip;
2275                 continue;
2276 skip:
2277                 if (likely(num))
2278                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2279                 num += skip_bufs;
2280                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2281                 sq->size -= num;
2282                 num = 0;
2283                 skip_bufs = 0;
2284         }
2285
2286         if (likely(num)) {
2287                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2288                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2289                 sq->size -= num;
2290         }
2291 }
2292
2293 /**
2294  * DPDK callback for transmit.
2295  *
2296  * @param txq
2297  *   Generic pointer transmit queue.
2298  * @param tx_pkts
2299  *   Packets to transmit.
2300  * @param nb_pkts
2301  *   Number of packets in array.
2302  *
2303  * @return
2304  *   Number of packets successfully transmitted.
2305  */
2306 static uint16_t
2307 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2308 {
2309         struct mrvl_txq *q = txq;
2310         struct mrvl_shadow_txq *sq;
2311         struct pp2_hif *hif;
2312         struct pp2_ppio_desc descs[nb_pkts];
2313         unsigned int core_id = rte_lcore_id();
2314         int i, ret, bytes_sent = 0;
2315         uint16_t num, sq_free_size;
2316         uint64_t addr;
2317
2318         hif = mrvl_get_hif(q->priv, core_id);
2319         sq = &q->shadow_txqs[core_id];
2320
2321         if (unlikely(!q->priv->ppio || !hif))
2322                 return 0;
2323
2324         if (sq->size)
2325                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2326                                        sq, q->queue_id, 0);
2327
2328         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2329         if (unlikely(nb_pkts > sq_free_size)) {
2330                 MRVL_LOG(DEBUG,
2331                         "No room in shadow queue for %d packets! %d packets will be sent.",
2332                         nb_pkts, sq_free_size);
2333                 nb_pkts = sq_free_size;
2334         }
2335
2336         for (i = 0; i < nb_pkts; i++) {
2337                 struct rte_mbuf *mbuf = tx_pkts[i];
2338                 int gen_l3_cksum, gen_l4_cksum;
2339                 enum pp2_outq_l3_type l3_type;
2340                 enum pp2_outq_l4_type l4_type;
2341
2342                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2343                         struct rte_mbuf *pref_pkt_hdr;
2344
2345                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2346                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2347                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2348                 }
2349
2350                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
2351                 sq->ent[sq->head].buff.addr =
2352                         rte_mbuf_data_iova_default(mbuf);
2353                 sq->ent[sq->head].bpool =
2354                         (unlikely(mbuf->port >= RTE_MAX_ETHPORTS ||
2355                          mbuf->refcnt > 1)) ? NULL :
2356                          mrvl_port_to_bpool_lookup[mbuf->port];
2357                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
2358                 sq->size++;
2359
2360                 pp2_ppio_outq_desc_reset(&descs[i]);
2361                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
2362                                                  rte_pktmbuf_iova(mbuf));
2363                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
2364                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
2365                                                rte_pktmbuf_pkt_len(mbuf));
2366
2367                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2368                 /*
2369                  * in case unsupported ol_flags were passed
2370                  * do not update descriptor offload information
2371                  */
2372                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
2373                                               &l3_type, &l4_type, &gen_l3_cksum,
2374                                               &gen_l4_cksum);
2375                 if (unlikely(ret))
2376                         continue;
2377
2378                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2379                                                   mbuf->l2_len,
2380                                                   mbuf->l2_len + mbuf->l3_len,
2381                                                   gen_l3_cksum, gen_l4_cksum);
2382         }
2383
2384         num = nb_pkts;
2385         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2386         /* number of packets that were not sent */
2387         if (unlikely(num > nb_pkts)) {
2388                 for (i = nb_pkts; i < num; i++) {
2389                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2390                                 MRVL_PP2_TX_SHADOWQ_MASK;
2391                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
2392                         bytes_sent -=
2393                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2394                 }
2395                 sq->size -= num - nb_pkts;
2396         }
2397
2398         q->bytes_sent += bytes_sent;
2399
2400         return nb_pkts;
2401 }
2402
2403 /**
2404  * Initialize packet processor.
2405  *
2406  * @return
2407  *   0 on success, negative error value otherwise.
2408  */
2409 static int
2410 mrvl_init_pp2(void)
2411 {
2412         struct pp2_init_params init_params;
2413
2414         memset(&init_params, 0, sizeof(init_params));
2415         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
2416         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
2417         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
2418
2419         return pp2_init(&init_params);
2420 }
2421
2422 /**
2423  * Deinitialize packet processor.
2424  *
2425  * @return
2426  *   0 on success, negative error value otherwise.
2427  */
2428 static void
2429 mrvl_deinit_pp2(void)
2430 {
2431         pp2_deinit();
2432 }
2433
2434 /**
2435  * Create private device structure.
2436  *
2437  * @param dev_name
2438  *   Pointer to the port name passed in the initialization parameters.
2439  *
2440  * @return
2441  *   Pointer to the newly allocated private device structure.
2442  */
2443 static struct mrvl_priv *
2444 mrvl_priv_create(const char *dev_name)
2445 {
2446         struct pp2_bpool_params bpool_params;
2447         char match[MRVL_MATCH_LEN];
2448         struct mrvl_priv *priv;
2449         int ret, bpool_bit;
2450
2451         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
2452         if (!priv)
2453                 return NULL;
2454
2455         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
2456                                        &priv->pp_id, &priv->ppio_id);
2457         if (ret)
2458                 goto out_free_priv;
2459
2460         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
2461                                      PP2_BPOOL_NUM_POOLS);
2462         if (bpool_bit < 0)
2463                 goto out_free_priv;
2464         priv->bpool_bit = bpool_bit;
2465
2466         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
2467                  priv->bpool_bit);
2468         memset(&bpool_params, 0, sizeof(bpool_params));
2469         bpool_params.match = match;
2470         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
2471         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
2472         if (ret)
2473                 goto out_clear_bpool_bit;
2474
2475         priv->ppio_params.type = PP2_PPIO_T_NIC;
2476         rte_spinlock_init(&priv->lock);
2477
2478         return priv;
2479 out_clear_bpool_bit:
2480         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2481 out_free_priv:
2482         rte_free(priv);
2483         return NULL;
2484 }
2485
2486 /**
2487  * Create device representing Ethernet port.
2488  *
2489  * @param name
2490  *   Pointer to the port's name.
2491  *
2492  * @return
2493  *   0 on success, negative error value otherwise.
2494  */
2495 static int
2496 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
2497 {
2498         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
2499         struct rte_eth_dev *eth_dev;
2500         struct mrvl_priv *priv;
2501         struct ifreq req;
2502
2503         eth_dev = rte_eth_dev_allocate(name);
2504         if (!eth_dev)
2505                 return -ENOMEM;
2506
2507         priv = mrvl_priv_create(name);
2508         if (!priv) {
2509                 ret = -ENOMEM;
2510                 goto out_free_dev;
2511         }
2512
2513         eth_dev->data->mac_addrs =
2514                 rte_zmalloc("mac_addrs",
2515                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2516         if (!eth_dev->data->mac_addrs) {
2517                 MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
2518                 ret = -ENOMEM;
2519                 goto out_free_priv;
2520         }
2521
2522         memset(&req, 0, sizeof(req));
2523         strcpy(req.ifr_name, name);
2524         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2525         if (ret)
2526                 goto out_free_mac;
2527
2528         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2529                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2530
2531         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2532         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
2533         eth_dev->data->kdrv = RTE_KDRV_NONE;
2534         eth_dev->data->dev_private = priv;
2535         eth_dev->device = &vdev->device;
2536         eth_dev->dev_ops = &mrvl_ops;
2537
2538         rte_eth_dev_probing_finish(eth_dev);
2539         return 0;
2540 out_free_mac:
2541         rte_free(eth_dev->data->mac_addrs);
2542 out_free_dev:
2543         rte_eth_dev_release_port(eth_dev);
2544 out_free_priv:
2545         rte_free(priv);
2546
2547         return ret;
2548 }
2549
2550 /**
2551  * Cleanup previously created device representing Ethernet port.
2552  *
2553  * @param name
2554  *   Pointer to the port name.
2555  */
2556 static void
2557 mrvl_eth_dev_destroy(const char *name)
2558 {
2559         struct rte_eth_dev *eth_dev;
2560         struct mrvl_priv *priv;
2561
2562         eth_dev = rte_eth_dev_allocated(name);
2563         if (!eth_dev)
2564                 return;
2565
2566         priv = eth_dev->data->dev_private;
2567         pp2_bpool_deinit(priv->bpool);
2568         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2569         rte_free(priv);
2570         rte_free(eth_dev->data->mac_addrs);
2571         rte_eth_dev_release_port(eth_dev);
2572 }
2573
2574 /**
2575  * Callback used by rte_kvargs_process() during argument parsing.
2576  *
2577  * @param key
2578  *   Pointer to the parsed key (unused).
2579  * @param value
2580  *   Pointer to the parsed value.
2581  * @param extra_args
2582  *   Pointer to the extra arguments which contains address of the
2583  *   table of pointers to parsed interface names.
2584  *
2585  * @return
2586  *   Always 0.
2587  */
2588 static int
2589 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2590                  void *extra_args)
2591 {
2592         struct mrvl_ifnames *ifnames = extra_args;
2593
2594         ifnames->names[ifnames->idx++] = value;
2595
2596         return 0;
2597 }
2598
2599 /**
2600  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2601  */
2602 static void
2603 mrvl_deinit_hifs(void)
2604 {
2605         int i;
2606
2607         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
2608                 if (hifs[i])
2609                         pp2_hif_deinit(hifs[i]);
2610         }
2611         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
2612         memset(hifs, 0, sizeof(hifs));
2613 }
2614
2615 /**
2616  * DPDK callback to register the virtual device.
2617  *
2618  * @param vdev
2619  *   Pointer to the virtual device.
2620  *
2621  * @return
2622  *   0 on success, negative error value otherwise.
2623  */
2624 static int
2625 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2626 {
2627         struct rte_kvargs *kvlist;
2628         struct mrvl_ifnames ifnames;
2629         int ret = -EINVAL;
2630         uint32_t i, ifnum, cfgnum;
2631         const char *params;
2632
2633         params = rte_vdev_device_args(vdev);
2634         if (!params)
2635                 return -EINVAL;
2636
2637         kvlist = rte_kvargs_parse(params, valid_args);
2638         if (!kvlist)
2639                 return -EINVAL;
2640
2641         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2642         if (ifnum > RTE_DIM(ifnames.names))
2643                 goto out_free_kvlist;
2644
2645         ifnames.idx = 0;
2646         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2647                            mrvl_get_ifnames, &ifnames);
2648
2649
2650         /*
2651          * The below system initialization should be done only once,
2652          * on the first provided configuration file
2653          */
2654         if (!mrvl_qos_cfg) {
2655                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2656                 MRVL_LOG(INFO, "Parsing config file!");
2657                 if (cfgnum > 1) {
2658                         MRVL_LOG(ERR, "Cannot handle more than one config file!");
2659                         goto out_free_kvlist;
2660                 } else if (cfgnum == 1) {
2661                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2662                                            mrvl_get_qoscfg, &mrvl_qos_cfg);
2663                 }
2664         }
2665
2666         if (mrvl_dev_num)
2667                 goto init_devices;
2668
2669         MRVL_LOG(INFO, "Perform MUSDK initializations");
2670
2671         ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist);
2672         if (ret)
2673                 goto out_free_kvlist;
2674
2675         ret = mrvl_init_pp2();
2676         if (ret) {
2677                 MRVL_LOG(ERR, "Failed to init PP!");
2678                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2679                 goto out_free_kvlist;
2680         }
2681
2682         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2683         memset(mrvl_port_to_bpool_lookup, 0, sizeof(mrvl_port_to_bpool_lookup));
2684
2685         mrvl_lcore_first = RTE_MAX_LCORE;
2686         mrvl_lcore_last = 0;
2687
2688 init_devices:
2689         for (i = 0; i < ifnum; i++) {
2690                 MRVL_LOG(INFO, "Creating %s", ifnames.names[i]);
2691                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
2692                 if (ret)
2693                         goto out_cleanup;
2694         }
2695         mrvl_dev_num += ifnum;
2696
2697         rte_kvargs_free(kvlist);
2698
2699         return 0;
2700 out_cleanup:
2701         for (; i > 0; i--)
2702                 mrvl_eth_dev_destroy(ifnames.names[i]);
2703
2704         if (mrvl_dev_num == 0) {
2705                 mrvl_deinit_pp2();
2706                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2707         }
2708 out_free_kvlist:
2709         rte_kvargs_free(kvlist);
2710
2711         return ret;
2712 }
2713
2714 /**
2715  * DPDK callback to remove virtual device.
2716  *
2717  * @param vdev
2718  *   Pointer to the removed virtual device.
2719  *
2720  * @return
2721  *   0 on success, negative error value otherwise.
2722  */
2723 static int
2724 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2725 {
2726         int i;
2727         const char *name;
2728
2729         name = rte_vdev_device_name(vdev);
2730         if (!name)
2731                 return -EINVAL;
2732
2733         MRVL_LOG(INFO, "Removing %s", name);
2734
2735         RTE_ETH_FOREACH_DEV(i) { /* FIXME: removing all devices! */
2736                 char ifname[RTE_ETH_NAME_MAX_LEN];
2737
2738                 rte_eth_dev_get_name_by_port(i, ifname);
2739                 mrvl_eth_dev_destroy(ifname);
2740                 mrvl_dev_num--;
2741         }
2742
2743         if (mrvl_dev_num == 0) {
2744                 MRVL_LOG(INFO, "Perform MUSDK deinit");
2745                 mrvl_deinit_hifs();
2746                 mrvl_deinit_pp2();
2747                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2748         }
2749
2750         return 0;
2751 }
2752
2753 static struct rte_vdev_driver pmd_mrvl_drv = {
2754         .probe = rte_pmd_mrvl_probe,
2755         .remove = rte_pmd_mrvl_remove,
2756 };
2757
2758 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv);
2759 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2);
2760
2761 RTE_INIT(mrvl_init_log)
2762 {
2763         mrvl_logtype = rte_log_register("pmd.net.mvpp2");
2764         if (mrvl_logtype >= 0)
2765                 rte_log_set_level(mrvl_logtype, RTE_LOG_NOTICE);
2766 }