drivers/net/mvpp2/mrvl_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Marvell International Ltd.
   3  * Copyright(c) 2017 Semihalf.
   4  * All rights reserved.
   5  */
   6
   7 #include <rte_ethdev_driver.h>
   8 #include <rte_kvargs.h>
   9 #include <rte_log.h>
  10 #include <rte_malloc.h>
  11 #include <rte_bus_vdev.h>
  12
  13 #include <fcntl.h>
  14 #include <linux/ethtool.h>
  15 #include <linux/sockios.h>
  16 #include <net/if.h>
  17 #include <net/if_arp.h>
  18 #include <sys/ioctl.h>
  19 #include <sys/socket.h>
  20 #include <sys/stat.h>
  21 #include <sys/types.h>
  22
  23 #include <rte_mvep_common.h>
  24 #include "mrvl_ethdev.h"
  25 #include "mrvl_qos.h"
  26 #include "mrvl_flow.h"
  27 #include "mrvl_mtr.h"
  28
  29 /* bitmask with reserved hifs */
  30 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  31 /* bitmask with reserved bpools */
  32 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  33 /* bitmask with reserved kernel RSS tables */
  34 #define MRVL_MUSDK_RSS_RESERVED 0x01
  35 /* maximum number of available hifs */
  36 #define MRVL_MUSDK_HIFS_MAX 9
  37
  38 /* prefetch shift */
  39 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  40
  41 /* TCAM has 25 entries reserved for uc/mc filter entries */
  42 #define MRVL_MAC_ADDRS_MAX 25
  43 #define MRVL_MATCH_LEN 16
  44 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  45 /* Maximum allowable packet size */
  46 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  47
  48 #define MRVL_IFACE_NAME_ARG "iface"
  49 #define MRVL_CFG_ARG "cfg"
  50
  51 #define MRVL_BURST_SIZE 64
  52
  53 #define MRVL_ARP_LENGTH 28
  54
  55 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  56
  57 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
  58 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
  59
  60 /** Port Rx offload capabilities */
  61 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
  62                           DEV_RX_OFFLOAD_JUMBO_FRAME | \
  63                           DEV_RX_OFFLOAD_CHECKSUM)
  64
  65 /** Port Tx offloads capabilities */
  66 #define MRVL_TX_OFFLOADS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
  67                           DEV_TX_OFFLOAD_UDP_CKSUM | \
  68                           DEV_TX_OFFLOAD_TCP_CKSUM)
  69
  70 static const char * const valid_args[] = {
  71         MRVL_IFACE_NAME_ARG,
  72         MRVL_CFG_ARG,
  73         NULL
  74 };
  75
  76 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
  77 static struct pp2_hif *hifs[RTE_MAX_LCORE];
  78 static int used_bpools[PP2_NUM_PKT_PROC] = {
  79         [0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED
  80 };
  81
  82 static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
  83 static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
  84 static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
  85
  86 int mrvl_logtype;
  87
  88 struct mrvl_ifnames {
  89         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
  90         int idx;
  91 };
  92
  93 /*
  94  * To use buffer harvesting based on loopback port shadow queue structure
  95  * was introduced for buffers information bookkeeping.
  96  *
  97  * Before sending the packet, related buffer information (pp2_buff_inf) is
  98  * stored in shadow queue. After packet is transmitted no longer used
  99  * packet buffer is released back to it's original hardware pool,
 100  * on condition it originated from interface.
 101  * In case it  was generated by application itself i.e: mbuf->port field is
 102  * 0xff then its released to software mempool.
 103  */
 104 struct mrvl_shadow_txq {
 105         int head;           /* write index - used when sending buffers */
 106         int tail;           /* read index - used when releasing buffers */
 107         u16 size;           /* queue occupied size */
 108         u16 num_to_release; /* number of buffers sent, that can be released */
 109         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 110 };
 111
 112 struct mrvl_rxq {
 113         struct mrvl_priv *priv;
 114         struct rte_mempool *mp;
 115         int queue_id;
 116         int port_id;
 117         int cksum_enabled;
 118         uint64_t bytes_recv;
 119         uint64_t drop_mac;
 120 };
 121
 122 struct mrvl_txq {
 123         struct mrvl_priv *priv;
 124         int queue_id;
 125         int port_id;
 126         uint64_t bytes_sent;
 127         struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE];
 128         int tx_deferred_start;
 129 };
 130
 131 static int mrvl_lcore_first;
 132 static int mrvl_lcore_last;
 133 static int mrvl_dev_num;
 134
 135 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
 136 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
 137                         struct pp2_hif *hif, unsigned int core_id,
 138                         struct mrvl_shadow_txq *sq, int qid, int force);
 139
 140 #define MRVL_XSTATS_TBL_ENTRY(name) { \
 141         #name, offsetof(struct pp2_ppio_statistics, name),      \
 142         sizeof(((struct pp2_ppio_statistics *)0)->name)         \
 143 }
 144
 145 /* Table with xstats data */
 146 static struct {
 147         const char *name;
 148         unsigned int offset;
 149         unsigned int size;
 150 } mrvl_xstats_tbl[] = {
 151         MRVL_XSTATS_TBL_ENTRY(rx_bytes),
 152         MRVL_XSTATS_TBL_ENTRY(rx_packets),
 153         MRVL_XSTATS_TBL_ENTRY(rx_unicast_packets),
 154         MRVL_XSTATS_TBL_ENTRY(rx_errors),
 155         MRVL_XSTATS_TBL_ENTRY(rx_fullq_dropped),
 156         MRVL_XSTATS_TBL_ENTRY(rx_bm_dropped),
 157         MRVL_XSTATS_TBL_ENTRY(rx_early_dropped),
 158         MRVL_XSTATS_TBL_ENTRY(rx_fifo_dropped),
 159         MRVL_XSTATS_TBL_ENTRY(rx_cls_dropped),
 160         MRVL_XSTATS_TBL_ENTRY(tx_bytes),
 161         MRVL_XSTATS_TBL_ENTRY(tx_packets),
 162         MRVL_XSTATS_TBL_ENTRY(tx_unicast_packets),
 163         MRVL_XSTATS_TBL_ENTRY(tx_errors)
 164 };
 165
 166 static inline int
 167 mrvl_get_bpool_size(int pp2_id, int pool_id)
 168 {
 169         int i;
 170         int size = 0;
 171
 172         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 173                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 174
 175         return size;
 176 }
 177
 178 static inline int
 179 mrvl_reserve_bit(int *bitmap, int max)
 180 {
 181         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 182
 183         if (n >= max)
 184                 return -1;
 185
 186         *bitmap |= 1 << n;
 187
 188         return n;
 189 }
 190
 191 static int
 192 mrvl_init_hif(int core_id)
 193 {
 194         struct pp2_hif_params params;
 195         char match[MRVL_MATCH_LEN];
 196         int ret;
 197
 198         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
 199         if (ret < 0) {
 200                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 201                 return ret;
 202         }
 203
 204         snprintf(match, sizeof(match), "hif-%d", ret);
 205         memset(&params, 0, sizeof(params));
 206         params.match = match;
 207         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
 208         ret = pp2_hif_init(&params, &hifs[core_id]);
 209         if (ret) {
 210                 MRVL_LOG(ERR, "Failed to initialize hif %d", core_id);
 211                 return ret;
 212         }
 213
 214         return 0;
 215 }
 216
 217 static inline struct pp2_hif*
 218 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
 219 {
 220         int ret;
 221
 222         if (likely(hifs[core_id] != NULL))
 223                 return hifs[core_id];
 224
 225         rte_spinlock_lock(&priv->lock);
 226
 227         ret = mrvl_init_hif(core_id);
 228         if (ret < 0) {
 229                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 230                 goto out;
 231         }
 232
 233         if (core_id < mrvl_lcore_first)
 234                 mrvl_lcore_first = core_id;
 235
 236         if (core_id > mrvl_lcore_last)
 237                 mrvl_lcore_last = core_id;
 238 out:
 239         rte_spinlock_unlock(&priv->lock);
 240
 241         return hifs[core_id];
 242 }
 243
 244 /**
 245  * Configure rss based on dpdk rss configuration.
 246  *
 247  * @param priv
 248  *   Pointer to private structure.
 249  * @param rss_conf
 250  *   Pointer to RSS configuration.
 251  *
 252  * @return
 253  *   0 on success, negative error value otherwise.
 254  */
 255 static int
 256 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 257 {
 258         if (rss_conf->rss_key)
 259                 MRVL_LOG(WARNING, "Changing hash key is not supported");
 260
 261         if (rss_conf->rss_hf == 0) {
 262                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 263         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 264                 priv->ppio_params.inqs_params.hash_type =
 265                         PP2_PPIO_HASH_T_2_TUPLE;
 266         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 267                 priv->ppio_params.inqs_params.hash_type =
 268                         PP2_PPIO_HASH_T_5_TUPLE;
 269                 priv->rss_hf_tcp = 1;
 270         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 271                 priv->ppio_params.inqs_params.hash_type =
 272                         PP2_PPIO_HASH_T_5_TUPLE;
 273                 priv->rss_hf_tcp = 0;
 274         } else {
 275                 return -EINVAL;
 276         }
 277
 278         return 0;
 279 }
 280
 281 /**
 282  * Ethernet device configuration.
 283  *
 284  * Prepare the driver for a given number of TX and RX queues and
 285  * configure RSS.
 286  *
 287  * @param dev
 288  *   Pointer to Ethernet device structure.
 289  *
 290  * @return
 291  *   0 on success, negative error value otherwise.
 292  */
 293 static int
 294 mrvl_dev_configure(struct rte_eth_dev *dev)
 295 {
 296         struct mrvl_priv *priv = dev->data->dev_private;
 297         int ret;
 298
 299         if (priv->ppio) {
 300                 MRVL_LOG(INFO, "Device reconfiguration is not supported");
 301                 return -EINVAL;
 302         }
 303
 304         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 305             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 306                 MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
 307                         dev->data->dev_conf.rxmode.mq_mode);
 308                 return -EINVAL;
 309         }
 310
 311         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 312                 MRVL_LOG(INFO, "Split headers not supported");
 313                 return -EINVAL;
 314         }
 315
 316         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 317                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 318                                  ETHER_HDR_LEN - ETHER_CRC_LEN;
 319
 320         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 321                                   dev->data->nb_rx_queues);
 322         if (ret < 0)
 323                 return ret;
 324
 325         ret = mrvl_configure_txqs(priv, dev->data->port_id,
 326                                   dev->data->nb_tx_queues);
 327         if (ret < 0)
 328                 return ret;
 329
 330         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 331         priv->ppio_params.maintain_stats = 1;
 332         priv->nb_rx_queues = dev->data->nb_rx_queues;
 333
 334         if (dev->data->nb_rx_queues == 1 &&
 335             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 336                 MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
 337                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 338
 339                 return 0;
 340         }
 341
 342         return mrvl_configure_rss(priv,
 343                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
 344 }
 345
 346 /**
 347  * DPDK callback to change the MTU.
 348  *
 349  * Setting the MTU affects hardware MRU (packets larger than the MRU
 350  * will be dropped).
 351  *
 352  * @param dev
 353  *   Pointer to Ethernet device structure.
 354  * @param mtu
 355  *   New MTU.
 356  *
 357  * @return
 358  *   0 on success, negative error value otherwise.
 359  */
 360 static int
 361 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 362 {
 363         struct mrvl_priv *priv = dev->data->dev_private;
 364         /* extra MV_MH_SIZE bytes are required for Marvell tag */
 365         uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
 366         int ret;
 367
 368         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
 369                 return -EINVAL;
 370
 371         if (!priv->ppio)
 372                 return 0;
 373
 374         ret = pp2_ppio_set_mru(priv->ppio, mru);
 375         if (ret)
 376                 return ret;
 377
 378         return pp2_ppio_set_mtu(priv->ppio, mtu);
 379 }
 380
 381 /**
 382  * DPDK callback to bring the link up.
 383  *
 384  * @param dev
 385  *   Pointer to Ethernet device structure.
 386  *
 387  * @return
 388  *   0 on success, negative error value otherwise.
 389  */
 390 static int
 391 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 392 {
 393         struct mrvl_priv *priv = dev->data->dev_private;
 394         int ret;
 395
 396         if (!priv->ppio)
 397                 return -EPERM;
 398
 399         ret = pp2_ppio_enable(priv->ppio);
 400         if (ret)
 401                 return ret;
 402
 403         /*
 404          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 405          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 406          * PP2_TRAFFIC_INGRESS_EGRESS.
 407          *
 408          * Set mtu to default DPDK value here.
 409          */
 410         ret = mrvl_mtu_set(dev, dev->data->mtu);
 411         if (ret)
 412                 pp2_ppio_disable(priv->ppio);
 413
 414         return ret;
 415 }
 416
 417 /**
 418  * DPDK callback to bring the link down.
 419  *
 420  * @param dev
 421  *   Pointer to Ethernet device structure.
 422  *
 423  * @return
 424  *   0 on success, negative error value otherwise.
 425  */
 426 static int
 427 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 428 {
 429         struct mrvl_priv *priv = dev->data->dev_private;
 430
 431         if (!priv->ppio)
 432                 return -EPERM;
 433
 434         return pp2_ppio_disable(priv->ppio);
 435 }
 436
 437 /**
 438  * DPDK callback to start tx queue.
 439  *
 440  * @param dev
 441  *   Pointer to Ethernet device structure.
 442  * @param queue_id
 443  *   Transmit queue index.
 444  *
 445  * @return
 446  *   0 on success, negative error value otherwise.
 447  */
 448 static int
 449 mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
 450 {
 451         struct mrvl_priv *priv = dev->data->dev_private;
 452         int ret;
 453
 454         if (!priv)
 455                 return -EPERM;
 456
 457         /* passing 1 enables given tx queue */
 458         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1);
 459         if (ret) {
 460                 MRVL_LOG(ERR, "Failed to start txq %d", queue_id);
 461                 return ret;
 462         }
 463
 464         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 465
 466         return 0;
 467 }
 468
 469 /**
 470  * DPDK callback to stop tx queue.
 471  *
 472  * @param dev
 473  *   Pointer to Ethernet device structure.
 474  * @param queue_id
 475  *   Transmit queue index.
 476  *
 477  * @return
 478  *   0 on success, negative error value otherwise.
 479  */
 480 static int
 481 mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
 482 {
 483         struct mrvl_priv *priv = dev->data->dev_private;
 484         int ret;
 485
 486         if (!priv->ppio)
 487                 return -EPERM;
 488
 489         /* passing 0 disables given tx queue */
 490         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0);
 491         if (ret) {
 492                 MRVL_LOG(ERR, "Failed to stop txq %d", queue_id);
 493                 return ret;
 494         }
 495
 496         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 497
 498         return 0;
 499 }
 500
 501 /**
 502  * DPDK callback to start the device.
 503  *
 504  * @param dev
 505  *   Pointer to Ethernet device structure.
 506  *
 507  * @return
 508  *   0 on success, negative errno value on failure.
 509  */
 510 static int
 511 mrvl_dev_start(struct rte_eth_dev *dev)
 512 {
 513         struct mrvl_priv *priv = dev->data->dev_private;
 514         char match[MRVL_MATCH_LEN];
 515         int ret = 0, i, def_init_size;
 516
 517         if (priv->ppio)
 518                 return mrvl_dev_set_link_up(dev);
 519
 520         snprintf(match, sizeof(match), "ppio-%d:%d",
 521                  priv->pp_id, priv->ppio_id);
 522         priv->ppio_params.match = match;
 523
 524         /*
 525          * Calculate the minimum bpool size for refill feature as follows:
 526          * 2 default burst sizes multiply by number of rx queues.
 527          * If the bpool size will be below this value, new buffers will
 528          * be added to the pool.
 529          */
 530         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 531
 532         /* In case initial bpool size configured in queues setup is
 533          * smaller than minimum size add more buffers
 534          */
 535         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
 536         if (priv->bpool_init_size < def_init_size) {
 537                 int buffs_to_add = def_init_size - priv->bpool_init_size;
 538
 539                 priv->bpool_init_size += buffs_to_add;
 540                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
 541                 if (ret)
 542                         MRVL_LOG(ERR, "Failed to add buffers to bpool");
 543         }
 544
 545         /*
 546          * Calculate the maximum bpool size for refill feature as follows:
 547          * maximum number of descriptors in rx queue multiply by number
 548          * of rx queues plus minimum bpool size.
 549          * In case the bpool size will exceed this value, superfluous buffers
 550          * will be removed
 551          */
 552         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
 553                                 priv->bpool_min_size;
 554
 555         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 556         if (ret) {
 557                 MRVL_LOG(ERR, "Failed to init ppio");
 558                 return ret;
 559         }
 560
 561         /*
 562          * In case there are some some stale uc/mc mac addresses flush them
 563          * here. It cannot be done during mrvl_dev_close() as port information
 564          * is already gone at that point (due to pp2_ppio_deinit() in
 565          * mrvl_dev_stop()).
 566          */
 567         if (!priv->uc_mc_flushed) {
 568                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 569                 if (ret) {
 570                         MRVL_LOG(ERR,
 571                                 "Failed to flush uc/mc filter list");
 572                         goto out;
 573                 }
 574                 priv->uc_mc_flushed = 1;
 575         }
 576
 577         if (!priv->vlan_flushed) {
 578                 ret = pp2_ppio_flush_vlan(priv->ppio);
 579                 if (ret) {
 580                         MRVL_LOG(ERR, "Failed to flush vlan list");
 581                         /*
 582                          * TODO
 583                          * once pp2_ppio_flush_vlan() is supported jump to out
 584                          * goto out;
 585                          */
 586                 }
 587                 priv->vlan_flushed = 1;
 588         }
 589
 590         /* For default QoS config, don't start classifier. */
 591         if (mrvl_qos_cfg) {
 592                 ret = mrvl_start_qos_mapping(priv);
 593                 if (ret) {
 594                         MRVL_LOG(ERR, "Failed to setup QoS mapping");
 595                         goto out;
 596                 }
 597         }
 598
 599         ret = mrvl_dev_set_link_up(dev);
 600         if (ret) {
 601                 MRVL_LOG(ERR, "Failed to set link up");
 602                 goto out;
 603         }
 604
 605         /* start tx queues */
 606         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 607                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 608
 609                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
 610
 611                 if (!txq->tx_deferred_start)
 612                         continue;
 613
 614                 /*
 615                  * All txqs are started by default. Stop them
 616                  * so that tx_deferred_start works as expected.
 617                  */
 618                 ret = mrvl_tx_queue_stop(dev, i);
 619                 if (ret)
 620                         goto out;
 621         }
 622
 623         mrvl_flow_init(dev);
 624         mrvl_mtr_init(dev);
 625
 626         return 0;
 627 out:
 628         MRVL_LOG(ERR, "Failed to start device");
 629         pp2_ppio_deinit(priv->ppio);
 630         return ret;
 631 }
 632
 633 /**
 634  * Flush receive queues.
 635  *
 636  * @param dev
 637  *   Pointer to Ethernet device structure.
 638  */
 639 static void
 640 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 641 {
 642         int i;
 643
 644         MRVL_LOG(INFO, "Flushing rx queues");
 645         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 646                 int ret, num;
 647
 648                 do {
 649                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 650                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 651
 652                         num = MRVL_PP2_RXD_MAX;
 653                         ret = pp2_ppio_recv(q->priv->ppio,
 654                                             q->priv->rxq_map[q->queue_id].tc,
 655                                             q->priv->rxq_map[q->queue_id].inq,
 656                                             descs, (uint16_t *)&num);
 657                 } while (ret == 0 && num);
 658         }
 659 }
 660
 661 /**
 662  * Flush transmit shadow queues.
 663  *
 664  * @param dev
 665  *   Pointer to Ethernet device structure.
 666  */
 667 static void
 668 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 669 {
 670         int i, j;
 671         struct mrvl_txq *txq;
 672
 673         MRVL_LOG(INFO, "Flushing tx shadow queues");
 674         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 675                 txq = (struct mrvl_txq *)dev->data->tx_queues[i];
 676
 677                 for (j = 0; j < RTE_MAX_LCORE; j++) {
 678                         struct mrvl_shadow_txq *sq;
 679
 680                         if (!hifs[j])
 681                                 continue;
 682
 683                         sq = &txq->shadow_txqs[j];
 684                         mrvl_free_sent_buffers(txq->priv->ppio,
 685                                 hifs[j], j, sq, txq->queue_id, 1);
 686                         while (sq->tail != sq->head) {
 687                                 uint64_t addr = cookie_addr_high |
 688                                         sq->ent[sq->tail].buff.cookie;
 689                                 rte_pktmbuf_free(
 690                                         (struct rte_mbuf *)addr);
 691                                 sq->tail = (sq->tail + 1) &
 692                                             MRVL_PP2_TX_SHADOWQ_MASK;
 693                         }
 694                         memset(sq, 0, sizeof(*sq));
 695                 }
 696         }
 697 }
 698
 699 /**
 700  * Flush hardware bpool (buffer-pool).
 701  *
 702  * @param dev
 703  *   Pointer to Ethernet device structure.
 704  */
 705 static void
 706 mrvl_flush_bpool(struct rte_eth_dev *dev)
 707 {
 708         struct mrvl_priv *priv = dev->data->dev_private;
 709         struct pp2_hif *hif;
 710         uint32_t num;
 711         int ret;
 712         unsigned int core_id = rte_lcore_id();
 713
 714         if (core_id == LCORE_ID_ANY)
 715                 core_id = 0;
 716
 717         hif = mrvl_get_hif(priv, core_id);
 718
 719         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 720         if (ret) {
 721                 MRVL_LOG(ERR, "Failed to get bpool buffers number");
 722                 return;
 723         }
 724
 725         while (num--) {
 726                 struct pp2_buff_inf inf;
 727                 uint64_t addr;
 728
 729                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
 730                 if (ret)
 731                         break;
 732
 733                 addr = cookie_addr_high | inf.cookie;
 734                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 735         }
 736 }
 737
 738 /**
 739  * DPDK callback to stop the device.
 740  *
 741  * @param dev
 742  *   Pointer to Ethernet device structure.
 743  */
 744 static void
 745 mrvl_dev_stop(struct rte_eth_dev *dev)
 746 {
 747         mrvl_dev_set_link_down(dev);
 748 }
 749
 750 /**
 751  * DPDK callback to close the device.
 752  *
 753  * @param dev
 754  *   Pointer to Ethernet device structure.
 755  */
 756 static void
 757 mrvl_dev_close(struct rte_eth_dev *dev)
 758 {
 759         struct mrvl_priv *priv = dev->data->dev_private;
 760         size_t i;
 761
 762         mrvl_flush_rx_queues(dev);
 763         mrvl_flush_tx_shadow_queues(dev);
 764         mrvl_flow_deinit(dev);
 765         mrvl_mtr_deinit(dev);
 766
 767         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 768                 struct pp2_ppio_tc_params *tc_params =
 769                         &priv->ppio_params.inqs_params.tcs_params[i];
 770
 771                 if (tc_params->inqs_params) {
 772                         rte_free(tc_params->inqs_params);
 773                         tc_params->inqs_params = NULL;
 774                 }
 775         }
 776
 777         if (priv->cls_tbl) {
 778                 pp2_cls_tbl_deinit(priv->cls_tbl);
 779                 priv->cls_tbl = NULL;
 780         }
 781
 782         if (priv->qos_tbl) {
 783                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 784                 priv->qos_tbl = NULL;
 785         }
 786
 787         mrvl_flush_bpool(dev);
 788
 789         if (priv->ppio) {
 790                 pp2_ppio_deinit(priv->ppio);
 791                 priv->ppio = NULL;
 792         }
 793
 794         /* policer must be released after ppio deinitialization */
 795         if (priv->default_policer) {
 796                 pp2_cls_plcr_deinit(priv->default_policer);
 797                 priv->default_policer = NULL;
 798         }
 799 }
 800
 801 /**
 802  * DPDK callback to retrieve physical link information.
 803  *
 804  * @param dev
 805  *   Pointer to Ethernet device structure.
 806  * @param wait_to_complete
 807  *   Wait for request completion (ignored).
 808  *
 809  * @return
 810  *   0 on success, negative error value otherwise.
 811  */
 812 static int
 813 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 814 {
 815         /*
 816          * TODO
 817          * once MUSDK provides necessary API use it here
 818          */
 819         struct mrvl_priv *priv = dev->data->dev_private;
 820         struct ethtool_cmd edata;
 821         struct ifreq req;
 822         int ret, fd, link_up;
 823
 824         if (!priv->ppio)
 825                 return -EPERM;
 826
 827         edata.cmd = ETHTOOL_GSET;
 828
 829         strcpy(req.ifr_name, dev->data->name);
 830         req.ifr_data = (void *)&edata;
 831
 832         fd = socket(AF_INET, SOCK_DGRAM, 0);
 833         if (fd == -1)
 834                 return -EFAULT;
 835
 836         ret = ioctl(fd, SIOCETHTOOL, &req);
 837         if (ret == -1) {
 838                 close(fd);
 839                 return -EFAULT;
 840         }
 841
 842         close(fd);
 843
 844         switch (ethtool_cmd_speed(&edata)) {
 845         case SPEED_10:
 846                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 847                 break;
 848         case SPEED_100:
 849                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 850                 break;
 851         case SPEED_1000:
 852                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 853                 break;
 854         case SPEED_10000:
 855                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 856                 break;
 857         default:
 858                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 859         }
 860
 861         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 862                                                          ETH_LINK_HALF_DUPLEX;
 863         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 864                                                            ETH_LINK_FIXED;
 865         pp2_ppio_get_link_state(priv->ppio, &link_up);
 866         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
 867
 868         return 0;
 869 }
 870
 871 /**
 872  * DPDK callback to enable promiscuous mode.
 873  *
 874  * @param dev
 875  *   Pointer to Ethernet device structure.
 876  */
 877 static void
 878 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
 879 {
 880         struct mrvl_priv *priv = dev->data->dev_private;
 881         int ret;
 882
 883         if (!priv->ppio)
 884                 return;
 885
 886         if (priv->isolated)
 887                 return;
 888
 889         ret = pp2_ppio_set_promisc(priv->ppio, 1);
 890         if (ret)
 891                 MRVL_LOG(ERR, "Failed to enable promiscuous mode");
 892 }
 893
 894 /**
 895  * DPDK callback to enable allmulti mode.
 896  *
 897  * @param dev
 898  *   Pointer to Ethernet device structure.
 899  */
 900 static void
 901 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
 902 {
 903         struct mrvl_priv *priv = dev->data->dev_private;
 904         int ret;
 905
 906         if (!priv->ppio)
 907                 return;
 908
 909         if (priv->isolated)
 910                 return;
 911
 912         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
 913         if (ret)
 914                 MRVL_LOG(ERR, "Failed enable all-multicast mode");
 915 }
 916
 917 /**
 918  * DPDK callback to disable promiscuous mode.
 919  *
 920  * @param dev
 921  *   Pointer to Ethernet device structure.
 922  */
 923 static void
 924 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
 925 {
 926         struct mrvl_priv *priv = dev->data->dev_private;
 927         int ret;
 928
 929         if (!priv->ppio)
 930                 return;
 931
 932         ret = pp2_ppio_set_promisc(priv->ppio, 0);
 933         if (ret)
 934                 MRVL_LOG(ERR, "Failed to disable promiscuous mode");
 935 }
 936
 937 /**
 938  * DPDK callback to disable allmulticast mode.
 939  *
 940  * @param dev
 941  *   Pointer to Ethernet device structure.
 942  */
 943 static void
 944 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
 945 {
 946         struct mrvl_priv *priv = dev->data->dev_private;
 947         int ret;
 948
 949         if (!priv->ppio)
 950                 return;
 951
 952         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
 953         if (ret)
 954                 MRVL_LOG(ERR, "Failed to disable all-multicast mode");
 955 }
 956
 957 /**
 958  * DPDK callback to remove a MAC address.
 959  *
 960  * @param dev
 961  *   Pointer to Ethernet device structure.
 962  * @param index
 963  *   MAC address index.
 964  */
 965 static void
 966 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 967 {
 968         struct mrvl_priv *priv = dev->data->dev_private;
 969         char buf[ETHER_ADDR_FMT_SIZE];
 970         int ret;
 971
 972         if (!priv->ppio)
 973                 return;
 974
 975         if (priv->isolated)
 976                 return;
 977
 978         ret = pp2_ppio_remove_mac_addr(priv->ppio,
 979                                        dev->data->mac_addrs[index].addr_bytes);
 980         if (ret) {
 981                 ether_format_addr(buf, sizeof(buf),
 982                                   &dev->data->mac_addrs[index]);
 983                 MRVL_LOG(ERR, "Failed to remove mac %s", buf);
 984         }
 985 }
 986
 987 /**
 988  * DPDK callback to add a MAC address.
 989  *
 990  * @param dev
 991  *   Pointer to Ethernet device structure.
 992  * @param mac_addr
 993  *   MAC address to register.
 994  * @param index
 995  *   MAC address index.
 996  * @param vmdq
 997  *   VMDq pool index to associate address with (unused).
 998  *
 999  * @return
1000  *   0 on success, negative error value otherwise.
1001  */
1002 static int
1003 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
1004                   uint32_t index, uint32_t vmdq __rte_unused)
1005 {
1006         struct mrvl_priv *priv = dev->data->dev_private;
1007         char buf[ETHER_ADDR_FMT_SIZE];
1008         int ret;
1009
1010         if (priv->isolated)
1011                 return -ENOTSUP;
1012
1013         if (index == 0)
1014                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
1015                 return -1;
1016
1017         if (!priv->ppio)
1018                 return 0;
1019
1020         /*
1021          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
1022          * parameter uc_filter_max. Maximum number of mc addresses is then
1023          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
1024          * 21 respectively.
1025          *
1026          * If more than uc_filter_max uc addresses were added to filter list
1027          * then NIC will switch to promiscuous mode automatically.
1028          *
1029          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
1030          * were added to filter list then NIC will switch to all-multicast mode
1031          * automatically.
1032          */
1033         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
1034         if (ret) {
1035                 ether_format_addr(buf, sizeof(buf), mac_addr);
1036                 MRVL_LOG(ERR, "Failed to add mac %s", buf);
1037                 return -1;
1038         }
1039
1040         return 0;
1041 }
1042
1043 /**
1044  * DPDK callback to set the primary MAC address.
1045  *
1046  * @param dev
1047  *   Pointer to Ethernet device structure.
1048  * @param mac_addr
1049  *   MAC address to register.
1050  *
1051  * @return
1052  *   0 on success, negative error value otherwise.
1053  */
1054 static int
1055 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1056 {
1057         struct mrvl_priv *priv = dev->data->dev_private;
1058         int ret;
1059
1060         if (!priv->ppio)
1061                 return 0;
1062
1063         if (priv->isolated)
1064                 return -ENOTSUP;
1065
1066         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
1067         if (ret) {
1068                 char buf[ETHER_ADDR_FMT_SIZE];
1069                 ether_format_addr(buf, sizeof(buf), mac_addr);
1070                 MRVL_LOG(ERR, "Failed to set mac to %s", buf);
1071         }
1072
1073         return ret;
1074 }
1075
1076 /**
1077  * DPDK callback to get device statistics.
1078  *
1079  * @param dev
1080  *   Pointer to Ethernet device structure.
1081  * @param stats
1082  *   Stats structure output buffer.
1083  *
1084  * @return
1085  *   0 on success, negative error value otherwise.
1086  */
1087 static int
1088 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1089 {
1090         struct mrvl_priv *priv = dev->data->dev_private;
1091         struct pp2_ppio_statistics ppio_stats;
1092         uint64_t drop_mac = 0;
1093         unsigned int i, idx, ret;
1094
1095         if (!priv->ppio)
1096                 return -EPERM;
1097
1098         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1099                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1100                 struct pp2_ppio_inq_statistics rx_stats;
1101
1102                 if (!rxq)
1103                         continue;
1104
1105                 idx = rxq->queue_id;
1106                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1107                         MRVL_LOG(ERR,
1108                                 "rx queue %d stats out of range (0 - %d)",
1109                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1110                         continue;
1111                 }
1112
1113                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
1114                                                   priv->rxq_map[idx].tc,
1115                                                   priv->rxq_map[idx].inq,
1116                                                   &rx_stats, 0);
1117                 if (unlikely(ret)) {
1118                         MRVL_LOG(ERR,
1119                                 "Failed to update rx queue %d stats", idx);
1120                         break;
1121                 }
1122
1123                 stats->q_ibytes[idx] = rxq->bytes_recv;
1124                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
1125                 stats->q_errors[idx] = rx_stats.drop_early +
1126                                        rx_stats.drop_fullq +
1127                                        rx_stats.drop_bm +
1128                                        rxq->drop_mac;
1129                 stats->ibytes += rxq->bytes_recv;
1130                 drop_mac += rxq->drop_mac;
1131         }
1132
1133         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1134                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1135                 struct pp2_ppio_outq_statistics tx_stats;
1136
1137                 if (!txq)
1138                         continue;
1139
1140                 idx = txq->queue_id;
1141                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1142                         MRVL_LOG(ERR,
1143                                 "tx queue %d stats out of range (0 - %d)",
1144                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1145                 }
1146
1147                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1148                                                    &tx_stats, 0);
1149                 if (unlikely(ret)) {
1150                         MRVL_LOG(ERR,
1151                                 "Failed to update tx queue %d stats", idx);
1152                         break;
1153                 }
1154
1155                 stats->q_opackets[idx] = tx_stats.deq_desc;
1156                 stats->q_obytes[idx] = txq->bytes_sent;
1157                 stats->obytes += txq->bytes_sent;
1158         }
1159
1160         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1161         if (unlikely(ret)) {
1162                 MRVL_LOG(ERR, "Failed to update port statistics");
1163                 return ret;
1164         }
1165
1166         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1167         stats->opackets += ppio_stats.tx_packets;
1168         stats->imissed += ppio_stats.rx_fullq_dropped +
1169                           ppio_stats.rx_bm_dropped +
1170                           ppio_stats.rx_early_dropped +
1171                           ppio_stats.rx_fifo_dropped +
1172                           ppio_stats.rx_cls_dropped;
1173         stats->ierrors = drop_mac;
1174
1175         return 0;
1176 }
1177
1178 /**
1179  * DPDK callback to clear device statistics.
1180  *
1181  * @param dev
1182  *   Pointer to Ethernet device structure.
1183  */
1184 static void
1185 mrvl_stats_reset(struct rte_eth_dev *dev)
1186 {
1187         struct mrvl_priv *priv = dev->data->dev_private;
1188         int i;
1189
1190         if (!priv->ppio)
1191                 return;
1192
1193         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1194                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1195
1196                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1197                                             priv->rxq_map[i].inq, NULL, 1);
1198                 rxq->bytes_recv = 0;
1199                 rxq->drop_mac = 0;
1200         }
1201
1202         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1203                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1204
1205                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1206                 txq->bytes_sent = 0;
1207         }
1208
1209         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1210 }
1211
1212 /**
1213  * DPDK callback to get extended statistics.
1214  *
1215  * @param dev
1216  *   Pointer to Ethernet device structure.
1217  * @param stats
1218  *   Pointer to xstats table.
1219  * @param n
1220  *   Number of entries in xstats table.
1221  * @return
1222  *   Negative value on error, number of read xstats otherwise.
1223  */
1224 static int
1225 mrvl_xstats_get(struct rte_eth_dev *dev,
1226                 struct rte_eth_xstat *stats, unsigned int n)
1227 {
1228         struct mrvl_priv *priv = dev->data->dev_private;
1229         struct pp2_ppio_statistics ppio_stats;
1230         unsigned int i;
1231
1232         if (!stats)
1233                 return 0;
1234
1235         pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1236         for (i = 0; i < n && i < RTE_DIM(mrvl_xstats_tbl); i++) {
1237                 uint64_t val;
1238
1239                 if (mrvl_xstats_tbl[i].size == sizeof(uint32_t))
1240                         val = *(uint32_t *)((uint8_t *)&ppio_stats +
1241                                             mrvl_xstats_tbl[i].offset);
1242                 else if (mrvl_xstats_tbl[i].size == sizeof(uint64_t))
1243                         val = *(uint64_t *)((uint8_t *)&ppio_stats +
1244                                             mrvl_xstats_tbl[i].offset);
1245                 else
1246                         return -EINVAL;
1247
1248                 stats[i].id = i;
1249                 stats[i].value = val;
1250         }
1251
1252         return n;
1253 }
1254
1255 /**
1256  * DPDK callback to reset extended statistics.
1257  *
1258  * @param dev
1259  *   Pointer to Ethernet device structure.
1260  */
1261 static void
1262 mrvl_xstats_reset(struct rte_eth_dev *dev)
1263 {
1264         mrvl_stats_reset(dev);
1265 }
1266
1267 /**
1268  * DPDK callback to get extended statistics names.
1269  *
1270  * @param dev (unused)
1271  *   Pointer to Ethernet device structure.
1272  * @param xstats_names
1273  *   Pointer to xstats names table.
1274  * @param size
1275  *   Size of the xstats names table.
1276  * @return
1277  *   Number of read names.
1278  */
1279 static int
1280 mrvl_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
1281                       struct rte_eth_xstat_name *xstats_names,
1282                       unsigned int size)
1283 {
1284         unsigned int i;
1285
1286         if (!xstats_names)
1287                 return RTE_DIM(mrvl_xstats_tbl);
1288
1289         for (i = 0; i < size && i < RTE_DIM(mrvl_xstats_tbl); i++)
1290                 snprintf(xstats_names[i].name, RTE_ETH_XSTATS_NAME_SIZE, "%s",
1291                          mrvl_xstats_tbl[i].name);
1292
1293         return size;
1294 }
1295
1296 /**
1297  * DPDK callback to get information about the device.
1298  *
1299  * @param dev
1300  *   Pointer to Ethernet device structure (unused).
1301  * @param info
1302  *   Info structure output buffer.
1303  */
1304 static void
1305 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
1306                    struct rte_eth_dev_info *info)
1307 {
1308         info->speed_capa = ETH_LINK_SPEED_10M |
1309                            ETH_LINK_SPEED_100M |
1310                            ETH_LINK_SPEED_1G |
1311                            ETH_LINK_SPEED_10G;
1312
1313         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1314         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1315         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1316
1317         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1318         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1319         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1320
1321         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1322         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1323         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1324
1325         info->rx_offload_capa = MRVL_RX_OFFLOADS;
1326         info->rx_queue_offload_capa = MRVL_RX_OFFLOADS;
1327
1328         info->tx_offload_capa = MRVL_TX_OFFLOADS;
1329         info->tx_queue_offload_capa = MRVL_TX_OFFLOADS;
1330
1331         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1332                                        ETH_RSS_NONFRAG_IPV4_TCP |
1333                                        ETH_RSS_NONFRAG_IPV4_UDP;
1334
1335         /* By default packets are dropped if no descriptors are available */
1336         info->default_rxconf.rx_drop_en = 1;
1337
1338         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1339 }
1340
1341 /**
1342  * Return supported packet types.
1343  *
1344  * @param dev
1345  *   Pointer to Ethernet device structure (unused).
1346  *
1347  * @return
1348  *   Const pointer to the table with supported packet types.
1349  */
1350 static const uint32_t *
1351 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1352 {
1353         static const uint32_t ptypes[] = {
1354                 RTE_PTYPE_L2_ETHER,
1355                 RTE_PTYPE_L2_ETHER_VLAN,
1356                 RTE_PTYPE_L2_ETHER_QINQ,
1357                 RTE_PTYPE_L3_IPV4,
1358                 RTE_PTYPE_L3_IPV4_EXT,
1359                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1360                 RTE_PTYPE_L3_IPV6,
1361                 RTE_PTYPE_L3_IPV6_EXT,
1362                 RTE_PTYPE_L2_ETHER_ARP,
1363                 RTE_PTYPE_L4_TCP,
1364                 RTE_PTYPE_L4_UDP
1365         };
1366
1367         return ptypes;
1368 }
1369
1370 /**
1371  * DPDK callback to get information about specific receive queue.
1372  *
1373  * @param dev
1374  *   Pointer to Ethernet device structure.
1375  * @param rx_queue_id
1376  *   Receive queue index.
1377  * @param qinfo
1378  *   Receive queue information structure.
1379  */
1380 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1381                               struct rte_eth_rxq_info *qinfo)
1382 {
1383         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1384         struct mrvl_priv *priv = dev->data->dev_private;
1385         int inq = priv->rxq_map[rx_queue_id].inq;
1386         int tc = priv->rxq_map[rx_queue_id].tc;
1387         struct pp2_ppio_tc_params *tc_params =
1388                 &priv->ppio_params.inqs_params.tcs_params[tc];
1389
1390         qinfo->mp = q->mp;
1391         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1392 }
1393
1394 /**
1395  * DPDK callback to get information about specific transmit queue.
1396  *
1397  * @param dev
1398  *   Pointer to Ethernet device structure.
1399  * @param tx_queue_id
1400  *   Transmit queue index.
1401  * @param qinfo
1402  *   Transmit queue information structure.
1403  */
1404 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1405                               struct rte_eth_txq_info *qinfo)
1406 {
1407         struct mrvl_priv *priv = dev->data->dev_private;
1408         struct mrvl_txq *txq = dev->data->tx_queues[tx_queue_id];
1409
1410         qinfo->nb_desc =
1411                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1412         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1413 }
1414
1415 /**
1416  * DPDK callback to Configure a VLAN filter.
1417  *
1418  * @param dev
1419  *   Pointer to Ethernet device structure.
1420  * @param vlan_id
1421  *   VLAN ID to filter.
1422  * @param on
1423  *   Toggle filter.
1424  *
1425  * @return
1426  *   0 on success, negative error value otherwise.
1427  */
1428 static int
1429 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1430 {
1431         struct mrvl_priv *priv = dev->data->dev_private;
1432
1433         if (!priv->ppio)
1434                 return -EPERM;
1435
1436         if (priv->isolated)
1437                 return -ENOTSUP;
1438
1439         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1440                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1441 }
1442
1443 /**
1444  * Release buffers to hardware bpool (buffer-pool)
1445  *
1446  * @param rxq
1447  *   Receive queue pointer.
1448  * @param num
1449  *   Number of buffers to release to bpool.
1450  *
1451  * @return
1452  *   0 on success, negative error value otherwise.
1453  */
1454 static int
1455 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1456 {
1457         struct buff_release_entry entries[MRVL_PP2_RXD_MAX];
1458         struct rte_mbuf *mbufs[MRVL_PP2_RXD_MAX];
1459         int i, ret;
1460         unsigned int core_id;
1461         struct pp2_hif *hif;
1462         struct pp2_bpool *bpool;
1463
1464         core_id = rte_lcore_id();
1465         if (core_id == LCORE_ID_ANY)
1466                 core_id = 0;
1467
1468         hif = mrvl_get_hif(rxq->priv, core_id);
1469         if (!hif)
1470                 return -1;
1471
1472         bpool = rxq->priv->bpool;
1473
1474         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1475         if (ret)
1476                 return ret;
1477
1478         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1479                 cookie_addr_high =
1480                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1481
1482         for (i = 0; i < num; i++) {
1483                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1484                         != cookie_addr_high) {
1485                         MRVL_LOG(ERR,
1486                                 "mbuf virtual addr high 0x%lx out of range",
1487                                 (uint64_t)mbufs[i] >> 32);
1488                         goto out;
1489                 }
1490
1491                 entries[i].buff.addr =
1492                         rte_mbuf_data_iova_default(mbufs[i]);
1493                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
1494                 entries[i].bpool = bpool;
1495         }
1496
1497         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1498         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1499
1500         if (i != num)
1501                 goto out;
1502
1503         return 0;
1504 out:
1505         for (; i < num; i++)
1506                 rte_pktmbuf_free(mbufs[i]);
1507
1508         return -1;
1509 }
1510
1511 /**
1512  * DPDK callback to configure the receive queue.
1513  *
1514  * @param dev
1515  *   Pointer to Ethernet device structure.
1516  * @param idx
1517  *   RX queue index.
1518  * @param desc
1519  *   Number of descriptors to configure in queue.
1520  * @param socket
1521  *   NUMA socket on which memory must be allocated.
1522  * @param conf
1523  *   Thresholds parameters.
1524  * @param mp
1525  *   Memory pool for buffer allocations.
1526  *
1527  * @return
1528  *   0 on success, negative error value otherwise.
1529  */
1530 static int
1531 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1532                     unsigned int socket,
1533                     const struct rte_eth_rxconf *conf,
1534                     struct rte_mempool *mp)
1535 {
1536         struct mrvl_priv *priv = dev->data->dev_private;
1537         struct mrvl_rxq *rxq;
1538         uint32_t min_size,
1539                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1540         int ret, tc, inq;
1541         uint64_t offloads;
1542
1543         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1544
1545         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1546                 /*
1547                  * Unknown TC mapping, mapping will not have a correct queue.
1548                  */
1549                 MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu",
1550                         idx, priv->ppio_id);
1551                 return -EFAULT;
1552         }
1553
1554         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
1555                    MRVL_PKT_EFFEC_OFFS;
1556         if (min_size < max_rx_pkt_len) {
1557                 MRVL_LOG(ERR,
1558                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.",
1559                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
1560                         MRVL_PKT_EFFEC_OFFS,
1561                         max_rx_pkt_len);
1562                 return -EINVAL;
1563         }
1564
1565         if (dev->data->rx_queues[idx]) {
1566                 rte_free(dev->data->rx_queues[idx]);
1567                 dev->data->rx_queues[idx] = NULL;
1568         }
1569
1570         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1571         if (!rxq)
1572                 return -ENOMEM;
1573
1574         rxq->priv = priv;
1575         rxq->mp = mp;
1576         rxq->cksum_enabled = offloads & DEV_RX_OFFLOAD_IPV4_CKSUM;
1577         rxq->queue_id = idx;
1578         rxq->port_id = dev->data->port_id;
1579         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1580
1581         tc = priv->rxq_map[rxq->queue_id].tc,
1582         inq = priv->rxq_map[rxq->queue_id].inq;
1583         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1584                 desc;
1585
1586         ret = mrvl_fill_bpool(rxq, desc);
1587         if (ret) {
1588                 rte_free(rxq);
1589                 return ret;
1590         }
1591
1592         priv->bpool_init_size += desc;
1593
1594         dev->data->rx_queues[idx] = rxq;
1595
1596         return 0;
1597 }
1598
1599 /**
1600  * DPDK callback to release the receive queue.
1601  *
1602  * @param rxq
1603  *   Generic receive queue pointer.
1604  */
1605 static void
1606 mrvl_rx_queue_release(void *rxq)
1607 {
1608         struct mrvl_rxq *q = rxq;
1609         struct pp2_ppio_tc_params *tc_params;
1610         int i, num, tc, inq;
1611         struct pp2_hif *hif;
1612         unsigned int core_id = rte_lcore_id();
1613
1614         if (core_id == LCORE_ID_ANY)
1615                 core_id = 0;
1616
1617         if (!q)
1618                 return;
1619
1620         hif = mrvl_get_hif(q->priv, core_id);
1621
1622         if (!hif)
1623                 return;
1624
1625         tc = q->priv->rxq_map[q->queue_id].tc;
1626         inq = q->priv->rxq_map[q->queue_id].inq;
1627         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1628         num = tc_params->inqs_params[inq].size;
1629         for (i = 0; i < num; i++) {
1630                 struct pp2_buff_inf inf;
1631                 uint64_t addr;
1632
1633                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
1634                 addr = cookie_addr_high | inf.cookie;
1635                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1636         }
1637
1638         rte_free(q);
1639 }
1640
1641 /**
1642  * DPDK callback to configure the transmit queue.
1643  *
1644  * @param dev
1645  *   Pointer to Ethernet device structure.
1646  * @param idx
1647  *   Transmit queue index.
1648  * @param desc
1649  *   Number of descriptors to configure in the queue.
1650  * @param socket
1651  *   NUMA socket on which memory must be allocated.
1652  * @param conf
1653  *   Tx queue configuration parameters.
1654  *
1655  * @return
1656  *   0 on success, negative error value otherwise.
1657  */
1658 static int
1659 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1660                     unsigned int socket,
1661                     const struct rte_eth_txconf *conf)
1662 {
1663         struct mrvl_priv *priv = dev->data->dev_private;
1664         struct mrvl_txq *txq;
1665
1666         if (dev->data->tx_queues[idx]) {
1667                 rte_free(dev->data->tx_queues[idx]);
1668                 dev->data->tx_queues[idx] = NULL;
1669         }
1670
1671         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1672         if (!txq)
1673                 return -ENOMEM;
1674
1675         txq->priv = priv;
1676         txq->queue_id = idx;
1677         txq->port_id = dev->data->port_id;
1678         txq->tx_deferred_start = conf->tx_deferred_start;
1679         dev->data->tx_queues[idx] = txq;
1680
1681         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1682
1683         return 0;
1684 }
1685
1686 /**
1687  * DPDK callback to release the transmit queue.
1688  *
1689  * @param txq
1690  *   Generic transmit queue pointer.
1691  */
1692 static void
1693 mrvl_tx_queue_release(void *txq)
1694 {
1695         struct mrvl_txq *q = txq;
1696
1697         if (!q)
1698                 return;
1699
1700         rte_free(q);
1701 }
1702
1703 /**
1704  * DPDK callback to get flow control configuration.
1705  *
1706  * @param dev
1707  *  Pointer to Ethernet device structure.
1708  * @param fc_conf
1709  *  Pointer to the flow control configuration.
1710  *
1711  * @return
1712  *  0 on success, negative error value otherwise.
1713  */
1714 static int
1715 mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1716 {
1717         struct mrvl_priv *priv = dev->data->dev_private;
1718         int ret, en;
1719
1720         if (!priv)
1721                 return -EPERM;
1722
1723         ret = pp2_ppio_get_rx_pause(priv->ppio, &en);
1724         if (ret) {
1725                 MRVL_LOG(ERR, "Failed to read rx pause state");
1726                 return ret;
1727         }
1728
1729         fc_conf->mode = en ? RTE_FC_RX_PAUSE : RTE_FC_NONE;
1730
1731         return 0;
1732 }
1733
1734 /**
1735  * DPDK callback to set flow control configuration.
1736  *
1737  * @param dev
1738  *  Pointer to Ethernet device structure.
1739  * @param fc_conf
1740  *  Pointer to the flow control configuration.
1741  *
1742  * @return
1743  *  0 on success, negative error value otherwise.
1744  */
1745 static int
1746 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1747 {
1748         struct mrvl_priv *priv = dev->data->dev_private;
1749
1750         if (!priv)
1751                 return -EPERM;
1752
1753         if (fc_conf->high_water ||
1754             fc_conf->low_water ||
1755             fc_conf->pause_time ||
1756             fc_conf->mac_ctrl_frame_fwd ||
1757             fc_conf->autoneg) {
1758                 MRVL_LOG(ERR, "Flowctrl parameter is not supported");
1759
1760                 return -EINVAL;
1761         }
1762
1763         if (fc_conf->mode == RTE_FC_NONE ||
1764             fc_conf->mode == RTE_FC_RX_PAUSE) {
1765                 int ret, en;
1766
1767                 en = fc_conf->mode == RTE_FC_NONE ? 0 : 1;
1768                 ret = pp2_ppio_set_rx_pause(priv->ppio, en);
1769                 if (ret)
1770                         MRVL_LOG(ERR,
1771                                 "Failed to change flowctrl on RX side");
1772
1773                 return ret;
1774         }
1775
1776         return 0;
1777 }
1778
1779 /**
1780  * Update RSS hash configuration
1781  *
1782  * @param dev
1783  *   Pointer to Ethernet device structure.
1784  * @param rss_conf
1785  *   Pointer to RSS configuration.
1786  *
1787  * @return
1788  *   0 on success, negative error value otherwise.
1789  */
1790 static int
1791 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1792                      struct rte_eth_rss_conf *rss_conf)
1793 {
1794         struct mrvl_priv *priv = dev->data->dev_private;
1795
1796         if (priv->isolated)
1797                 return -ENOTSUP;
1798
1799         return mrvl_configure_rss(priv, rss_conf);
1800 }
1801
1802 /**
1803  * DPDK callback to get RSS hash configuration.
1804  *
1805  * @param dev
1806  *   Pointer to Ethernet device structure.
1807  * @rss_conf
1808  *   Pointer to RSS configuration.
1809  *
1810  * @return
1811  *   Always 0.
1812  */
1813 static int
1814 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1815                        struct rte_eth_rss_conf *rss_conf)
1816 {
1817         struct mrvl_priv *priv = dev->data->dev_private;
1818         enum pp2_ppio_hash_type hash_type =
1819                 priv->ppio_params.inqs_params.hash_type;
1820
1821         rss_conf->rss_key = NULL;
1822
1823         if (hash_type == PP2_PPIO_HASH_T_NONE)
1824                 rss_conf->rss_hf = 0;
1825         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1826                 rss_conf->rss_hf = ETH_RSS_IPV4;
1827         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1828                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1829         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1830                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1831
1832         return 0;
1833 }
1834
1835 /**
1836  * DPDK callback to get rte_flow callbacks.
1837  *
1838  * @param dev
1839  *   Pointer to the device structure.
1840  * @param filer_type
1841  *   Flow filter type.
1842  * @param filter_op
1843  *   Flow filter operation.
1844  * @param arg
1845  *   Pointer to pass the flow ops.
1846  *
1847  * @return
1848  *   0 on success, negative error value otherwise.
1849  */
1850 static int
1851 mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
1852                      enum rte_filter_type filter_type,
1853                      enum rte_filter_op filter_op, void *arg)
1854 {
1855         switch (filter_type) {
1856         case RTE_ETH_FILTER_GENERIC:
1857                 if (filter_op != RTE_ETH_FILTER_GET)
1858                         return -EINVAL;
1859                 *(const void **)arg = &mrvl_flow_ops;
1860                 return 0;
1861         default:
1862                 MRVL_LOG(WARNING, "Filter type (%d) not supported",
1863                                 filter_type);
1864                 return -EINVAL;
1865         }
1866 }
1867
1868 /**
1869  * DPDK callback to get rte_mtr callbacks.
1870  *
1871  * @param dev
1872  *   Pointer to the device structure.
1873  * @param ops
1874  *   Pointer to pass the mtr ops.
1875  *
1876  * @return
1877  *   Always 0.
1878  */
1879 static int
1880 mrvl_mtr_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
1881 {
1882         *(const void **)ops = &mrvl_mtr_ops;
1883
1884         return 0;
1885 }
1886
1887 static const struct eth_dev_ops mrvl_ops = {
1888         .dev_configure = mrvl_dev_configure,
1889         .dev_start = mrvl_dev_start,
1890         .dev_stop = mrvl_dev_stop,
1891         .dev_set_link_up = mrvl_dev_set_link_up,
1892         .dev_set_link_down = mrvl_dev_set_link_down,
1893         .dev_close = mrvl_dev_close,
1894         .link_update = mrvl_link_update,
1895         .promiscuous_enable = mrvl_promiscuous_enable,
1896         .allmulticast_enable = mrvl_allmulticast_enable,
1897         .promiscuous_disable = mrvl_promiscuous_disable,
1898         .allmulticast_disable = mrvl_allmulticast_disable,
1899         .mac_addr_remove = mrvl_mac_addr_remove,
1900         .mac_addr_add = mrvl_mac_addr_add,
1901         .mac_addr_set = mrvl_mac_addr_set,
1902         .mtu_set = mrvl_mtu_set,
1903         .stats_get = mrvl_stats_get,
1904         .stats_reset = mrvl_stats_reset,
1905         .xstats_get = mrvl_xstats_get,
1906         .xstats_reset = mrvl_xstats_reset,
1907         .xstats_get_names = mrvl_xstats_get_names,
1908         .dev_infos_get = mrvl_dev_infos_get,
1909         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
1910         .rxq_info_get = mrvl_rxq_info_get,
1911         .txq_info_get = mrvl_txq_info_get,
1912         .vlan_filter_set = mrvl_vlan_filter_set,
1913         .tx_queue_start = mrvl_tx_queue_start,
1914         .tx_queue_stop = mrvl_tx_queue_stop,
1915         .rx_queue_setup = mrvl_rx_queue_setup,
1916         .rx_queue_release = mrvl_rx_queue_release,
1917         .tx_queue_setup = mrvl_tx_queue_setup,
1918         .tx_queue_release = mrvl_tx_queue_release,
1919         .flow_ctrl_get = mrvl_flow_ctrl_get,
1920         .flow_ctrl_set = mrvl_flow_ctrl_set,
1921         .rss_hash_update = mrvl_rss_hash_update,
1922         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1923         .filter_ctrl = mrvl_eth_filter_ctrl,
1924         .mtr_ops_get = mrvl_mtr_ops_get,
1925 };
1926
1927 /**
1928  * Return packet type information and l3/l4 offsets.
1929  *
1930  * @param desc
1931  *   Pointer to the received packet descriptor.
1932  * @param l3_offset
1933  *   l3 packet offset.
1934  * @param l4_offset
1935  *   l4 packet offset.
1936  *
1937  * @return
1938  *   Packet type information.
1939  */
1940 static inline uint64_t
1941 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
1942                                     uint8_t *l3_offset, uint8_t *l4_offset)
1943 {
1944         enum pp2_inq_l3_type l3_type;
1945         enum pp2_inq_l4_type l4_type;
1946         enum pp2_inq_vlan_tag vlan_tag;
1947         uint64_t packet_type;
1948
1949         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
1950         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
1951         pp2_ppio_inq_desc_get_vlan_tag(desc, &vlan_tag);
1952
1953         packet_type = RTE_PTYPE_L2_ETHER;
1954
1955         switch (vlan_tag) {
1956         case PP2_INQ_VLAN_TAG_SINGLE:
1957                 packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
1958                 break;
1959         case PP2_INQ_VLAN_TAG_DOUBLE:
1960         case PP2_INQ_VLAN_TAG_TRIPLE:
1961                 packet_type |= RTE_PTYPE_L2_ETHER_QINQ;
1962                 break;
1963         default:
1964                 break;
1965         }
1966
1967         switch (l3_type) {
1968         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
1969                 packet_type |= RTE_PTYPE_L3_IPV4;
1970                 break;
1971         case PP2_INQ_L3_TYPE_IPV4_OK:
1972                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
1973                 break;
1974         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
1975                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
1976                 break;
1977         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
1978                 packet_type |= RTE_PTYPE_L3_IPV6;
1979                 break;
1980         case PP2_INQ_L3_TYPE_IPV6_EXT:
1981                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
1982                 break;
1983         case PP2_INQ_L3_TYPE_ARP:
1984                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
1985                 /*
1986                  * In case of ARP l4_offset is set to wrong value.
1987                  * Set it to proper one so that later on mbuf->l3_len can be
1988                  * calculated subtracting l4_offset and l3_offset.
1989                  */
1990                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
1991                 break;
1992         default:
1993                 MRVL_LOG(DEBUG, "Failed to recognise l3 packet type");
1994                 break;
1995         }
1996
1997         switch (l4_type) {
1998         case PP2_INQ_L4_TYPE_TCP:
1999                 packet_type |= RTE_PTYPE_L4_TCP;
2000                 break;
2001         case PP2_INQ_L4_TYPE_UDP:
2002                 packet_type |= RTE_PTYPE_L4_UDP;
2003                 break;
2004         default:
2005                 MRVL_LOG(DEBUG, "Failed to recognise l4 packet type");
2006                 break;
2007         }
2008
2009         return packet_type;
2010 }
2011
2012 /**
2013  * Get offload information from the received packet descriptor.
2014  *
2015  * @param desc
2016  *   Pointer to the received packet descriptor.
2017  *
2018  * @return
2019  *   Mbuf offload flags.
2020  */
2021 static inline uint64_t
2022 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
2023 {
2024         uint64_t flags;
2025         enum pp2_inq_desc_status status;
2026
2027         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
2028         if (unlikely(status != PP2_DESC_ERR_OK))
2029                 flags = PKT_RX_IP_CKSUM_BAD;
2030         else
2031                 flags = PKT_RX_IP_CKSUM_GOOD;
2032
2033         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
2034         if (unlikely(status != PP2_DESC_ERR_OK))
2035                 flags |= PKT_RX_L4_CKSUM_BAD;
2036         else
2037                 flags |= PKT_RX_L4_CKSUM_GOOD;
2038
2039         return flags;
2040 }
2041
2042 /**
2043  * DPDK callback for receive.
2044  *
2045  * @param rxq
2046  *   Generic pointer to the receive queue.
2047  * @param rx_pkts
2048  *   Array to store received packets.
2049  * @param nb_pkts
2050  *   Maximum number of packets in array.
2051  *
2052  * @return
2053  *   Number of packets successfully received.
2054  */
2055 static uint16_t
2056 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2057 {
2058         struct mrvl_rxq *q = rxq;
2059         struct pp2_ppio_desc descs[nb_pkts];
2060         struct pp2_bpool *bpool;
2061         int i, ret, rx_done = 0;
2062         int num;
2063         struct pp2_hif *hif;
2064         unsigned int core_id = rte_lcore_id();
2065
2066         hif = mrvl_get_hif(q->priv, core_id);
2067
2068         if (unlikely(!q->priv->ppio || !hif))
2069                 return 0;
2070
2071         bpool = q->priv->bpool;
2072
2073         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
2074                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
2075         if (unlikely(ret < 0)) {
2076                 MRVL_LOG(ERR, "Failed to receive packets");
2077                 return 0;
2078         }
2079         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
2080
2081         for (i = 0; i < nb_pkts; i++) {
2082                 struct rte_mbuf *mbuf;
2083                 uint8_t l3_offset, l4_offset;
2084                 enum pp2_inq_desc_status status;
2085                 uint64_t addr;
2086
2087                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2088                         struct pp2_ppio_desc *pref_desc;
2089                         u64 pref_addr;
2090
2091                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
2092                         pref_addr = cookie_addr_high |
2093                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
2094                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
2095                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
2096                 }
2097
2098                 addr = cookie_addr_high |
2099                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
2100                 mbuf = (struct rte_mbuf *)addr;
2101                 rte_pktmbuf_reset(mbuf);
2102
2103                 /* drop packet in case of mac, overrun or resource error */
2104                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
2105                 if (unlikely(status != PP2_DESC_ERR_OK)) {
2106                         struct pp2_buff_inf binf = {
2107                                 .addr = rte_mbuf_data_iova_default(mbuf),
2108                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
2109                         };
2110
2111                         pp2_bpool_put_buff(hif, bpool, &binf);
2112                         mrvl_port_bpool_size
2113                                 [bpool->pp2_id][bpool->id][core_id]++;
2114                         q->drop_mac++;
2115                         continue;
2116                 }
2117
2118                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
2119                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
2120                 mbuf->data_len = mbuf->pkt_len;
2121                 mbuf->port = q->port_id;
2122                 mbuf->packet_type =
2123                         mrvl_desc_to_packet_type_and_offset(&descs[i],
2124                                                             &l3_offset,
2125                                                             &l4_offset);
2126                 mbuf->l2_len = l3_offset;
2127                 mbuf->l3_len = l4_offset - l3_offset;
2128
2129                 if (likely(q->cksum_enabled))
2130                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
2131
2132                 rx_pkts[rx_done++] = mbuf;
2133                 q->bytes_recv += mbuf->pkt_len;
2134         }
2135
2136         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
2137                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
2138
2139                 if (unlikely(num <= q->priv->bpool_min_size ||
2140                              (!rx_done && num < q->priv->bpool_init_size))) {
2141                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
2142                         if (ret)
2143                                 MRVL_LOG(ERR, "Failed to fill bpool");
2144                 } else if (unlikely(num > q->priv->bpool_max_size)) {
2145                         int i;
2146                         int pkt_to_remove = num - q->priv->bpool_init_size;
2147                         struct rte_mbuf *mbuf;
2148                         struct pp2_buff_inf buff;
2149
2150                         MRVL_LOG(DEBUG,
2151                                 "port-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)",
2152                                 bpool->pp2_id, q->priv->ppio->port_id,
2153                                 bpool->id, pkt_to_remove, num,
2154                                 q->priv->bpool_init_size);
2155
2156                         for (i = 0; i < pkt_to_remove; i++) {
2157                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
2158                                 if (ret)
2159                                         break;
2160                                 mbuf = (struct rte_mbuf *)
2161                                         (cookie_addr_high | buff.cookie);
2162                                 rte_pktmbuf_free(mbuf);
2163                         }
2164                         mrvl_port_bpool_size
2165                                 [bpool->pp2_id][bpool->id][core_id] -= i;
2166                 }
2167                 rte_spinlock_unlock(&q->priv->lock);
2168         }
2169
2170         return rx_done;
2171 }
2172
2173 /**
2174  * Prepare offload information.
2175  *
2176  * @param ol_flags
2177  *   Offload flags.
2178  * @param packet_type
2179  *   Packet type bitfield.
2180  * @param l3_type
2181  *   Pointer to the pp2_ouq_l3_type structure.
2182  * @param l4_type
2183  *   Pointer to the pp2_outq_l4_type structure.
2184  * @param gen_l3_cksum
2185  *   Will be set to 1 in case l3 checksum is computed.
2186  * @param l4_cksum
2187  *   Will be set to 1 in case l4 checksum is computed.
2188  *
2189  * @return
2190  *   0 on success, negative error value otherwise.
2191  */
2192 static inline int
2193 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
2194                         enum pp2_outq_l3_type *l3_type,
2195                         enum pp2_outq_l4_type *l4_type,
2196                         int *gen_l3_cksum,
2197                         int *gen_l4_cksum)
2198 {
2199         /*
2200          * Based on ol_flags prepare information
2201          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
2202          * for offloading.
2203          */
2204         if (ol_flags & PKT_TX_IPV4) {
2205                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
2206                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
2207         } else if (ol_flags & PKT_TX_IPV6) {
2208                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
2209                 /* no checksum for ipv6 header */
2210                 *gen_l3_cksum = 0;
2211         } else {
2212                 /* if something different then stop processing */
2213                 return -1;
2214         }
2215
2216         ol_flags &= PKT_TX_L4_MASK;
2217         if ((packet_type & RTE_PTYPE_L4_TCP) &&
2218             ol_flags == PKT_TX_TCP_CKSUM) {
2219                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
2220                 *gen_l4_cksum = 1;
2221         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
2222                    ol_flags == PKT_TX_UDP_CKSUM) {
2223                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
2224                 *gen_l4_cksum = 1;
2225         } else {
2226                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
2227                 /* no checksum for other type */
2228                 *gen_l4_cksum = 0;
2229         }
2230
2231         return 0;
2232 }
2233
2234 /**
2235  * Release already sent buffers to bpool (buffer-pool).
2236  *
2237  * @param ppio
2238  *   Pointer to the port structure.
2239  * @param hif
2240  *   Pointer to the MUSDK hardware interface.
2241  * @param sq
2242  *   Pointer to the shadow queue.
2243  * @param qid
2244  *   Queue id number.
2245  * @param force
2246  *   Force releasing packets.
2247  */
2248 static inline void
2249 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
2250                        unsigned int core_id, struct mrvl_shadow_txq *sq,
2251                        int qid, int force)
2252 {
2253         struct buff_release_entry *entry;
2254         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
2255         int i;
2256
2257         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
2258
2259         sq->num_to_release += nb_done;
2260
2261         if (likely(!force &&
2262                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
2263                 return;
2264
2265         nb_done = sq->num_to_release;
2266         sq->num_to_release = 0;
2267
2268         for (i = 0; i < nb_done; i++) {
2269                 entry = &sq->ent[sq->tail + num];
2270                 if (unlikely(!entry->buff.addr)) {
2271                         MRVL_LOG(ERR,
2272                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!",
2273                                 sq->tail, (u64)entry->buff.cookie,
2274                                 (u64)entry->buff.addr);
2275                         skip_bufs = 1;
2276                         goto skip;
2277                 }
2278
2279                 if (unlikely(!entry->bpool)) {
2280                         struct rte_mbuf *mbuf;
2281
2282                         mbuf = (struct rte_mbuf *)
2283                                (cookie_addr_high | entry->buff.cookie);
2284                         rte_pktmbuf_free(mbuf);
2285                         skip_bufs = 1;
2286                         goto skip;
2287                 }
2288
2289                 mrvl_port_bpool_size
2290                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
2291                 num++;
2292                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
2293                         goto skip;
2294                 continue;
2295 skip:
2296                 if (likely(num))
2297                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2298                 num += skip_bufs;
2299                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2300                 sq->size -= num;
2301                 num = 0;
2302                 skip_bufs = 0;
2303         }
2304
2305         if (likely(num)) {
2306                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2307                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2308                 sq->size -= num;
2309         }
2310 }
2311
2312 /**
2313  * DPDK callback for transmit.
2314  *
2315  * @param txq
2316  *   Generic pointer transmit queue.
2317  * @param tx_pkts
2318  *   Packets to transmit.
2319  * @param nb_pkts
2320  *   Number of packets in array.
2321  *
2322  * @return
2323  *   Number of packets successfully transmitted.
2324  */
2325 static uint16_t
2326 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2327 {
2328         struct mrvl_txq *q = txq;
2329         struct mrvl_shadow_txq *sq;
2330         struct pp2_hif *hif;
2331         struct pp2_ppio_desc descs[nb_pkts];
2332         unsigned int core_id = rte_lcore_id();
2333         int i, ret, bytes_sent = 0;
2334         uint16_t num, sq_free_size;
2335         uint64_t addr;
2336
2337         hif = mrvl_get_hif(q->priv, core_id);
2338         sq = &q->shadow_txqs[core_id];
2339
2340         if (unlikely(!q->priv->ppio || !hif))
2341                 return 0;
2342
2343         if (sq->size)
2344                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2345                                        sq, q->queue_id, 0);
2346
2347         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2348         if (unlikely(nb_pkts > sq_free_size)) {
2349                 MRVL_LOG(DEBUG,
2350                         "No room in shadow queue for %d packets! %d packets will be sent.",
2351                         nb_pkts, sq_free_size);
2352                 nb_pkts = sq_free_size;
2353         }
2354
2355         for (i = 0; i < nb_pkts; i++) {
2356                 struct rte_mbuf *mbuf = tx_pkts[i];
2357                 int gen_l3_cksum, gen_l4_cksum;
2358                 enum pp2_outq_l3_type l3_type;
2359                 enum pp2_outq_l4_type l4_type;
2360
2361                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2362                         struct rte_mbuf *pref_pkt_hdr;
2363
2364                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2365                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2366                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2367                 }
2368
2369                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
2370                 sq->ent[sq->head].buff.addr =
2371                         rte_mbuf_data_iova_default(mbuf);
2372                 sq->ent[sq->head].bpool =
2373                         (unlikely(mbuf->port >= RTE_MAX_ETHPORTS ||
2374                          mbuf->refcnt > 1)) ? NULL :
2375                          mrvl_port_to_bpool_lookup[mbuf->port];
2376                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
2377                 sq->size++;
2378
2379                 pp2_ppio_outq_desc_reset(&descs[i]);
2380                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
2381                                                  rte_pktmbuf_iova(mbuf));
2382                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
2383                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
2384                                                rte_pktmbuf_pkt_len(mbuf));
2385
2386                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2387                 /*
2388                  * in case unsupported ol_flags were passed
2389                  * do not update descriptor offload information
2390                  */
2391                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
2392                                               &l3_type, &l4_type, &gen_l3_cksum,
2393                                               &gen_l4_cksum);
2394                 if (unlikely(ret))
2395                         continue;
2396
2397                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2398                                                   mbuf->l2_len,
2399                                                   mbuf->l2_len + mbuf->l3_len,
2400                                                   gen_l3_cksum, gen_l4_cksum);
2401         }
2402
2403         num = nb_pkts;
2404         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2405         /* number of packets that were not sent */
2406         if (unlikely(num > nb_pkts)) {
2407                 for (i = nb_pkts; i < num; i++) {
2408                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2409                                 MRVL_PP2_TX_SHADOWQ_MASK;
2410                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
2411                         bytes_sent -=
2412                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2413                 }
2414                 sq->size -= num - nb_pkts;
2415         }
2416
2417         q->bytes_sent += bytes_sent;
2418
2419         return nb_pkts;
2420 }
2421
2422 /**
2423  * Initialize packet processor.
2424  *
2425  * @return
2426  *   0 on success, negative error value otherwise.
2427  */
2428 static int
2429 mrvl_init_pp2(void)
2430 {
2431         struct pp2_init_params init_params;
2432
2433         memset(&init_params, 0, sizeof(init_params));
2434         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
2435         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
2436         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
2437
2438         return pp2_init(&init_params);
2439 }
2440
2441 /**
2442  * Deinitialize packet processor.
2443  *
2444  * @return
2445  *   0 on success, negative error value otherwise.
2446  */
2447 static void
2448 mrvl_deinit_pp2(void)
2449 {
2450         pp2_deinit();
2451 }
2452
2453 /**
2454  * Create private device structure.
2455  *
2456  * @param dev_name
2457  *   Pointer to the port name passed in the initialization parameters.
2458  *
2459  * @return
2460  *   Pointer to the newly allocated private device structure.
2461  */
2462 static struct mrvl_priv *
2463 mrvl_priv_create(const char *dev_name)
2464 {
2465         struct pp2_bpool_params bpool_params;
2466         char match[MRVL_MATCH_LEN];
2467         struct mrvl_priv *priv;
2468         int ret, bpool_bit;
2469
2470         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
2471         if (!priv)
2472                 return NULL;
2473
2474         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
2475                                        &priv->pp_id, &priv->ppio_id);
2476         if (ret)
2477                 goto out_free_priv;
2478
2479         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
2480                                      PP2_BPOOL_NUM_POOLS);
2481         if (bpool_bit < 0)
2482                 goto out_free_priv;
2483         priv->bpool_bit = bpool_bit;
2484
2485         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
2486                  priv->bpool_bit);
2487         memset(&bpool_params, 0, sizeof(bpool_params));
2488         bpool_params.match = match;
2489         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
2490         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
2491         if (ret)
2492                 goto out_clear_bpool_bit;
2493
2494         priv->ppio_params.type = PP2_PPIO_T_NIC;
2495         rte_spinlock_init(&priv->lock);
2496
2497         return priv;
2498 out_clear_bpool_bit:
2499         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2500 out_free_priv:
2501         rte_free(priv);
2502         return NULL;
2503 }
2504
2505 /**
2506  * Create device representing Ethernet port.
2507  *
2508  * @param name
2509  *   Pointer to the port's name.
2510  *
2511  * @return
2512  *   0 on success, negative error value otherwise.
2513  */
2514 static int
2515 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
2516 {
2517         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
2518         struct rte_eth_dev *eth_dev;
2519         struct mrvl_priv *priv;
2520         struct ifreq req;
2521
2522         eth_dev = rte_eth_dev_allocate(name);
2523         if (!eth_dev)
2524                 return -ENOMEM;
2525
2526         priv = mrvl_priv_create(name);
2527         if (!priv) {
2528                 ret = -ENOMEM;
2529                 goto out_free_dev;
2530         }
2531
2532         eth_dev->data->mac_addrs =
2533                 rte_zmalloc("mac_addrs",
2534                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2535         if (!eth_dev->data->mac_addrs) {
2536                 MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
2537                 ret = -ENOMEM;
2538                 goto out_free_priv;
2539         }
2540
2541         memset(&req, 0, sizeof(req));
2542         strcpy(req.ifr_name, name);
2543         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2544         if (ret)
2545                 goto out_free_mac;
2546
2547         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2548                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2549
2550         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2551         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
2552         eth_dev->data->kdrv = RTE_KDRV_NONE;
2553         eth_dev->data->dev_private = priv;
2554         eth_dev->device = &vdev->device;
2555         eth_dev->dev_ops = &mrvl_ops;
2556
2557         rte_eth_dev_probing_finish(eth_dev);
2558         return 0;
2559 out_free_mac:
2560         rte_free(eth_dev->data->mac_addrs);
2561 out_free_dev:
2562         rte_eth_dev_release_port(eth_dev);
2563 out_free_priv:
2564         rte_free(priv);
2565
2566         return ret;
2567 }
2568
2569 /**
2570  * Cleanup previously created device representing Ethernet port.
2571  *
2572  * @param name
2573  *   Pointer to the port name.
2574  */
2575 static void
2576 mrvl_eth_dev_destroy(const char *name)
2577 {
2578         struct rte_eth_dev *eth_dev;
2579         struct mrvl_priv *priv;
2580
2581         eth_dev = rte_eth_dev_allocated(name);
2582         if (!eth_dev)
2583                 return;
2584
2585         priv = eth_dev->data->dev_private;
2586         pp2_bpool_deinit(priv->bpool);
2587         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2588         rte_free(priv);
2589         rte_free(eth_dev->data->mac_addrs);
2590         rte_eth_dev_release_port(eth_dev);
2591 }
2592
2593 /**
2594  * Callback used by rte_kvargs_process() during argument parsing.
2595  *
2596  * @param key
2597  *   Pointer to the parsed key (unused).
2598  * @param value
2599  *   Pointer to the parsed value.
2600  * @param extra_args
2601  *   Pointer to the extra arguments which contains address of the
2602  *   table of pointers to parsed interface names.
2603  *
2604  * @return
2605  *   Always 0.
2606  */
2607 static int
2608 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2609                  void *extra_args)
2610 {
2611         struct mrvl_ifnames *ifnames = extra_args;
2612
2613         ifnames->names[ifnames->idx++] = value;
2614
2615         return 0;
2616 }
2617
2618 /**
2619  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2620  */
2621 static void
2622 mrvl_deinit_hifs(void)
2623 {
2624         int i;
2625
2626         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
2627                 if (hifs[i])
2628                         pp2_hif_deinit(hifs[i]);
2629         }
2630         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
2631         memset(hifs, 0, sizeof(hifs));
2632 }
2633
2634 /**
2635  * DPDK callback to register the virtual device.
2636  *
2637  * @param vdev
2638  *   Pointer to the virtual device.
2639  *
2640  * @return
2641  *   0 on success, negative error value otherwise.
2642  */
2643 static int
2644 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2645 {
2646         struct rte_kvargs *kvlist;
2647         struct mrvl_ifnames ifnames;
2648         int ret = -EINVAL;
2649         uint32_t i, ifnum, cfgnum;
2650         const char *params;
2651
2652         params = rte_vdev_device_args(vdev);
2653         if (!params)
2654                 return -EINVAL;
2655
2656         kvlist = rte_kvargs_parse(params, valid_args);
2657         if (!kvlist)
2658                 return -EINVAL;
2659
2660         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2661         if (ifnum > RTE_DIM(ifnames.names))
2662                 goto out_free_kvlist;
2663
2664         ifnames.idx = 0;
2665         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2666                            mrvl_get_ifnames, &ifnames);
2667
2668
2669         /*
2670          * The below system initialization should be done only once,
2671          * on the first provided configuration file
2672          */
2673         if (!mrvl_qos_cfg) {
2674                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2675                 MRVL_LOG(INFO, "Parsing config file!");
2676                 if (cfgnum > 1) {
2677                         MRVL_LOG(ERR, "Cannot handle more than one config file!");
2678                         goto out_free_kvlist;
2679                 } else if (cfgnum == 1) {
2680                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2681                                            mrvl_get_qoscfg, &mrvl_qos_cfg);
2682                 }
2683         }
2684
2685         if (mrvl_dev_num)
2686                 goto init_devices;
2687
2688         MRVL_LOG(INFO, "Perform MUSDK initializations");
2689
2690         ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist);
2691         if (ret)
2692                 goto out_free_kvlist;
2693
2694         ret = mrvl_init_pp2();
2695         if (ret) {
2696                 MRVL_LOG(ERR, "Failed to init PP!");
2697                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2698                 goto out_free_kvlist;
2699         }
2700
2701         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2702         memset(mrvl_port_to_bpool_lookup, 0, sizeof(mrvl_port_to_bpool_lookup));
2703
2704         mrvl_lcore_first = RTE_MAX_LCORE;
2705         mrvl_lcore_last = 0;
2706
2707 init_devices:
2708         for (i = 0; i < ifnum; i++) {
2709                 MRVL_LOG(INFO, "Creating %s", ifnames.names[i]);
2710                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
2711                 if (ret)
2712                         goto out_cleanup;
2713         }
2714         mrvl_dev_num += ifnum;
2715
2716         rte_kvargs_free(kvlist);
2717
2718         return 0;
2719 out_cleanup:
2720         for (; i > 0; i--)
2721                 mrvl_eth_dev_destroy(ifnames.names[i]);
2722
2723         if (mrvl_dev_num == 0) {
2724                 mrvl_deinit_pp2();
2725                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2726         }
2727 out_free_kvlist:
2728         rte_kvargs_free(kvlist);
2729
2730         return ret;
2731 }
2732
2733 /**
2734  * DPDK callback to remove virtual device.
2735  *
2736  * @param vdev
2737  *   Pointer to the removed virtual device.
2738  *
2739  * @return
2740  *   0 on success, negative error value otherwise.
2741  */
2742 static int
2743 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2744 {
2745         int i;
2746         const char *name;
2747
2748         name = rte_vdev_device_name(vdev);
2749         if (!name)
2750                 return -EINVAL;
2751
2752         MRVL_LOG(INFO, "Removing %s", name);
2753
2754         RTE_ETH_FOREACH_DEV(i) { /* FIXME: removing all devices! */
2755                 char ifname[RTE_ETH_NAME_MAX_LEN];
2756
2757                 rte_eth_dev_get_name_by_port(i, ifname);
2758                 mrvl_eth_dev_destroy(ifname);
2759                 mrvl_dev_num--;
2760         }
2761
2762         if (mrvl_dev_num == 0) {
2763                 MRVL_LOG(INFO, "Perform MUSDK deinit");
2764                 mrvl_deinit_hifs();
2765                 mrvl_deinit_pp2();
2766                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2767         }
2768
2769         return 0;
2770 }
2771
2772 static struct rte_vdev_driver pmd_mrvl_drv = {
2773         .probe = rte_pmd_mrvl_probe,
2774         .remove = rte_pmd_mrvl_remove,
2775 };
2776
2777 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv);
2778 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2);
2779
2780 RTE_INIT(mrvl_init_log)
2781 {
2782         mrvl_logtype = rte_log_register("pmd.net.mvpp2");
2783         if (mrvl_logtype >= 0)
2784                 rte_log_set_level(mrvl_logtype, RTE_LOG_NOTICE);
2785 }