drivers/net/mvpp2/mrvl_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Marvell International Ltd.
   3  * Copyright(c) 2017 Semihalf.
   4  * All rights reserved.
   5  */
   6
   7 #include <rte_ethdev_driver.h>
   8 #include <rte_kvargs.h>
   9 #include <rte_log.h>
  10 #include <rte_malloc.h>
  11 #include <rte_bus_vdev.h>
  12
  13 #include <fcntl.h>
  14 #include <linux/ethtool.h>
  15 #include <linux/sockios.h>
  16 #include <net/if.h>
  17 #include <net/if_arp.h>
  18 #include <sys/ioctl.h>
  19 #include <sys/socket.h>
  20 #include <sys/stat.h>
  21 #include <sys/types.h>
  22
  23 #include <rte_mvep_common.h>
  24 #include "mrvl_ethdev.h"
  25 #include "mrvl_qos.h"
  26 #include "mrvl_flow.h"
  27 #include "mrvl_mtr.h"
  28 #include "mrvl_tm.h"
  29
  30 /* bitmask with reserved hifs */
  31 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  32 /* bitmask with reserved bpools */
  33 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  34 /* bitmask with reserved kernel RSS tables */
  35 #define MRVL_MUSDK_RSS_RESERVED 0x01
  36 /* maximum number of available hifs */
  37 #define MRVL_MUSDK_HIFS_MAX 9
  38
  39 /* prefetch shift */
  40 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  41
  42 /* TCAM has 25 entries reserved for uc/mc filter entries */
  43 #define MRVL_MAC_ADDRS_MAX 25
  44 #define MRVL_MATCH_LEN 16
  45 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  46 /* Maximum allowable packet size */
  47 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  48
  49 #define MRVL_IFACE_NAME_ARG "iface"
  50 #define MRVL_CFG_ARG "cfg"
  51
  52 #define MRVL_BURST_SIZE 64
  53
  54 #define MRVL_ARP_LENGTH 28
  55
  56 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  57
  58 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
  59 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
  60
  61 /** Port Rx offload capabilities */
  62 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
  63                           DEV_RX_OFFLOAD_JUMBO_FRAME | \
  64                           DEV_RX_OFFLOAD_CHECKSUM)
  65
  66 /** Port Tx offloads capabilities */
  67 #define MRVL_TX_OFFLOADS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
  68                           DEV_TX_OFFLOAD_UDP_CKSUM | \
  69                           DEV_TX_OFFLOAD_TCP_CKSUM)
  70
  71 static const char * const valid_args[] = {
  72         MRVL_IFACE_NAME_ARG,
  73         MRVL_CFG_ARG,
  74         NULL
  75 };
  76
  77 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
  78 static struct pp2_hif *hifs[RTE_MAX_LCORE];
  79 static int used_bpools[PP2_NUM_PKT_PROC] = {
  80         [0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED
  81 };
  82
  83 static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
  84 static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
  85 static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
  86
  87 int mrvl_logtype;
  88
  89 struct mrvl_ifnames {
  90         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
  91         int idx;
  92 };
  93
  94 /*
  95  * To use buffer harvesting based on loopback port shadow queue structure
  96  * was introduced for buffers information bookkeeping.
  97  *
  98  * Before sending the packet, related buffer information (pp2_buff_inf) is
  99  * stored in shadow queue. After packet is transmitted no longer used
 100  * packet buffer is released back to it's original hardware pool,
 101  * on condition it originated from interface.
 102  * In case it  was generated by application itself i.e: mbuf->port field is
 103  * 0xff then its released to software mempool.
 104  */
 105 struct mrvl_shadow_txq {
 106         int head;           /* write index - used when sending buffers */
 107         int tail;           /* read index - used when releasing buffers */
 108         u16 size;           /* queue occupied size */
 109         u16 num_to_release; /* number of buffers sent, that can be released */
 110         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 111 };
 112
 113 struct mrvl_rxq {
 114         struct mrvl_priv *priv;
 115         struct rte_mempool *mp;
 116         int queue_id;
 117         int port_id;
 118         int cksum_enabled;
 119         uint64_t bytes_recv;
 120         uint64_t drop_mac;
 121 };
 122
 123 struct mrvl_txq {
 124         struct mrvl_priv *priv;
 125         int queue_id;
 126         int port_id;
 127         uint64_t bytes_sent;
 128         struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE];
 129         int tx_deferred_start;
 130 };
 131
 132 static int mrvl_lcore_first;
 133 static int mrvl_lcore_last;
 134 static int mrvl_dev_num;
 135
 136 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
 137 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
 138                         struct pp2_hif *hif, unsigned int core_id,
 139                         struct mrvl_shadow_txq *sq, int qid, int force);
 140
 141 #define MRVL_XSTATS_TBL_ENTRY(name) { \
 142         #name, offsetof(struct pp2_ppio_statistics, name),      \
 143         sizeof(((struct pp2_ppio_statistics *)0)->name)         \
 144 }
 145
 146 /* Table with xstats data */
 147 static struct {
 148         const char *name;
 149         unsigned int offset;
 150         unsigned int size;
 151 } mrvl_xstats_tbl[] = {
 152         MRVL_XSTATS_TBL_ENTRY(rx_bytes),
 153         MRVL_XSTATS_TBL_ENTRY(rx_packets),
 154         MRVL_XSTATS_TBL_ENTRY(rx_unicast_packets),
 155         MRVL_XSTATS_TBL_ENTRY(rx_errors),
 156         MRVL_XSTATS_TBL_ENTRY(rx_fullq_dropped),
 157         MRVL_XSTATS_TBL_ENTRY(rx_bm_dropped),
 158         MRVL_XSTATS_TBL_ENTRY(rx_early_dropped),
 159         MRVL_XSTATS_TBL_ENTRY(rx_fifo_dropped),
 160         MRVL_XSTATS_TBL_ENTRY(rx_cls_dropped),
 161         MRVL_XSTATS_TBL_ENTRY(tx_bytes),
 162         MRVL_XSTATS_TBL_ENTRY(tx_packets),
 163         MRVL_XSTATS_TBL_ENTRY(tx_unicast_packets),
 164         MRVL_XSTATS_TBL_ENTRY(tx_errors)
 165 };
 166
 167 static inline int
 168 mrvl_get_bpool_size(int pp2_id, int pool_id)
 169 {
 170         int i;
 171         int size = 0;
 172
 173         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 174                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 175
 176         return size;
 177 }
 178
 179 static inline int
 180 mrvl_reserve_bit(int *bitmap, int max)
 181 {
 182         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 183
 184         if (n >= max)
 185                 return -1;
 186
 187         *bitmap |= 1 << n;
 188
 189         return n;
 190 }
 191
 192 static int
 193 mrvl_init_hif(int core_id)
 194 {
 195         struct pp2_hif_params params;
 196         char match[MRVL_MATCH_LEN];
 197         int ret;
 198
 199         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
 200         if (ret < 0) {
 201                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 202                 return ret;
 203         }
 204
 205         snprintf(match, sizeof(match), "hif-%d", ret);
 206         memset(&params, 0, sizeof(params));
 207         params.match = match;
 208         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
 209         ret = pp2_hif_init(&params, &hifs[core_id]);
 210         if (ret) {
 211                 MRVL_LOG(ERR, "Failed to initialize hif %d", core_id);
 212                 return ret;
 213         }
 214
 215         return 0;
 216 }
 217
 218 static inline struct pp2_hif*
 219 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
 220 {
 221         int ret;
 222
 223         if (likely(hifs[core_id] != NULL))
 224                 return hifs[core_id];
 225
 226         rte_spinlock_lock(&priv->lock);
 227
 228         ret = mrvl_init_hif(core_id);
 229         if (ret < 0) {
 230                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 231                 goto out;
 232         }
 233
 234         if (core_id < mrvl_lcore_first)
 235                 mrvl_lcore_first = core_id;
 236
 237         if (core_id > mrvl_lcore_last)
 238                 mrvl_lcore_last = core_id;
 239 out:
 240         rte_spinlock_unlock(&priv->lock);
 241
 242         return hifs[core_id];
 243 }
 244
 245 /**
 246  * Configure rss based on dpdk rss configuration.
 247  *
 248  * @param priv
 249  *   Pointer to private structure.
 250  * @param rss_conf
 251  *   Pointer to RSS configuration.
 252  *
 253  * @return
 254  *   0 on success, negative error value otherwise.
 255  */
 256 static int
 257 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 258 {
 259         if (rss_conf->rss_key)
 260                 MRVL_LOG(WARNING, "Changing hash key is not supported");
 261
 262         if (rss_conf->rss_hf == 0) {
 263                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 264         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 265                 priv->ppio_params.inqs_params.hash_type =
 266                         PP2_PPIO_HASH_T_2_TUPLE;
 267         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 268                 priv->ppio_params.inqs_params.hash_type =
 269                         PP2_PPIO_HASH_T_5_TUPLE;
 270                 priv->rss_hf_tcp = 1;
 271         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 272                 priv->ppio_params.inqs_params.hash_type =
 273                         PP2_PPIO_HASH_T_5_TUPLE;
 274                 priv->rss_hf_tcp = 0;
 275         } else {
 276                 return -EINVAL;
 277         }
 278
 279         return 0;
 280 }
 281
 282 /**
 283  * Ethernet device configuration.
 284  *
 285  * Prepare the driver for a given number of TX and RX queues and
 286  * configure RSS.
 287  *
 288  * @param dev
 289  *   Pointer to Ethernet device structure.
 290  *
 291  * @return
 292  *   0 on success, negative error value otherwise.
 293  */
 294 static int
 295 mrvl_dev_configure(struct rte_eth_dev *dev)
 296 {
 297         struct mrvl_priv *priv = dev->data->dev_private;
 298         int ret;
 299
 300         if (priv->ppio) {
 301                 MRVL_LOG(INFO, "Device reconfiguration is not supported");
 302                 return -EINVAL;
 303         }
 304
 305         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 306             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 307                 MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
 308                         dev->data->dev_conf.rxmode.mq_mode);
 309                 return -EINVAL;
 310         }
 311
 312         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 313                 MRVL_LOG(INFO, "Split headers not supported");
 314                 return -EINVAL;
 315         }
 316
 317         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 318                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 319                                  ETHER_HDR_LEN - ETHER_CRC_LEN;
 320
 321         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 322                                   dev->data->nb_rx_queues);
 323         if (ret < 0)
 324                 return ret;
 325
 326         ret = mrvl_configure_txqs(priv, dev->data->port_id,
 327                                   dev->data->nb_tx_queues);
 328         if (ret < 0)
 329                 return ret;
 330
 331         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 332         priv->ppio_params.maintain_stats = 1;
 333         priv->nb_rx_queues = dev->data->nb_rx_queues;
 334
 335         ret = mrvl_tm_init(dev);
 336         if (ret < 0)
 337                 return ret;
 338
 339         if (dev->data->nb_rx_queues == 1 &&
 340             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 341                 MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
 342                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 343
 344                 return 0;
 345         }
 346
 347         return mrvl_configure_rss(priv,
 348                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
 349 }
 350
 351 /**
 352  * DPDK callback to change the MTU.
 353  *
 354  * Setting the MTU affects hardware MRU (packets larger than the MRU
 355  * will be dropped).
 356  *
 357  * @param dev
 358  *   Pointer to Ethernet device structure.
 359  * @param mtu
 360  *   New MTU.
 361  *
 362  * @return
 363  *   0 on success, negative error value otherwise.
 364  */
 365 static int
 366 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 367 {
 368         struct mrvl_priv *priv = dev->data->dev_private;
 369         /* extra MV_MH_SIZE bytes are required for Marvell tag */
 370         uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
 371         int ret;
 372
 373         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
 374                 return -EINVAL;
 375
 376         if (!priv->ppio)
 377                 return 0;
 378
 379         ret = pp2_ppio_set_mru(priv->ppio, mru);
 380         if (ret)
 381                 return ret;
 382
 383         return pp2_ppio_set_mtu(priv->ppio, mtu);
 384 }
 385
 386 /**
 387  * DPDK callback to bring the link up.
 388  *
 389  * @param dev
 390  *   Pointer to Ethernet device structure.
 391  *
 392  * @return
 393  *   0 on success, negative error value otherwise.
 394  */
 395 static int
 396 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 397 {
 398         struct mrvl_priv *priv = dev->data->dev_private;
 399         int ret;
 400
 401         if (!priv->ppio)
 402                 return -EPERM;
 403
 404         ret = pp2_ppio_enable(priv->ppio);
 405         if (ret)
 406                 return ret;
 407
 408         /*
 409          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 410          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 411          * PP2_TRAFFIC_INGRESS_EGRESS.
 412          *
 413          * Set mtu to default DPDK value here.
 414          */
 415         ret = mrvl_mtu_set(dev, dev->data->mtu);
 416         if (ret)
 417                 pp2_ppio_disable(priv->ppio);
 418
 419         return ret;
 420 }
 421
 422 /**
 423  * DPDK callback to bring the link down.
 424  *
 425  * @param dev
 426  *   Pointer to Ethernet device structure.
 427  *
 428  * @return
 429  *   0 on success, negative error value otherwise.
 430  */
 431 static int
 432 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 433 {
 434         struct mrvl_priv *priv = dev->data->dev_private;
 435
 436         if (!priv->ppio)
 437                 return -EPERM;
 438
 439         return pp2_ppio_disable(priv->ppio);
 440 }
 441
 442 /**
 443  * DPDK callback to start tx queue.
 444  *
 445  * @param dev
 446  *   Pointer to Ethernet device structure.
 447  * @param queue_id
 448  *   Transmit queue index.
 449  *
 450  * @return
 451  *   0 on success, negative error value otherwise.
 452  */
 453 static int
 454 mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
 455 {
 456         struct mrvl_priv *priv = dev->data->dev_private;
 457         int ret;
 458
 459         if (!priv)
 460                 return -EPERM;
 461
 462         /* passing 1 enables given tx queue */
 463         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1);
 464         if (ret) {
 465                 MRVL_LOG(ERR, "Failed to start txq %d", queue_id);
 466                 return ret;
 467         }
 468
 469         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 470
 471         return 0;
 472 }
 473
 474 /**
 475  * DPDK callback to stop tx queue.
 476  *
 477  * @param dev
 478  *   Pointer to Ethernet device structure.
 479  * @param queue_id
 480  *   Transmit queue index.
 481  *
 482  * @return
 483  *   0 on success, negative error value otherwise.
 484  */
 485 static int
 486 mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
 487 {
 488         struct mrvl_priv *priv = dev->data->dev_private;
 489         int ret;
 490
 491         if (!priv->ppio)
 492                 return -EPERM;
 493
 494         /* passing 0 disables given tx queue */
 495         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0);
 496         if (ret) {
 497                 MRVL_LOG(ERR, "Failed to stop txq %d", queue_id);
 498                 return ret;
 499         }
 500
 501         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 502
 503         return 0;
 504 }
 505
 506 /**
 507  * DPDK callback to start the device.
 508  *
 509  * @param dev
 510  *   Pointer to Ethernet device structure.
 511  *
 512  * @return
 513  *   0 on success, negative errno value on failure.
 514  */
 515 static int
 516 mrvl_dev_start(struct rte_eth_dev *dev)
 517 {
 518         struct mrvl_priv *priv = dev->data->dev_private;
 519         char match[MRVL_MATCH_LEN];
 520         int ret = 0, i, def_init_size;
 521
 522         if (priv->ppio)
 523                 return mrvl_dev_set_link_up(dev);
 524
 525         snprintf(match, sizeof(match), "ppio-%d:%d",
 526                  priv->pp_id, priv->ppio_id);
 527         priv->ppio_params.match = match;
 528
 529         /*
 530          * Calculate the minimum bpool size for refill feature as follows:
 531          * 2 default burst sizes multiply by number of rx queues.
 532          * If the bpool size will be below this value, new buffers will
 533          * be added to the pool.
 534          */
 535         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 536
 537         /* In case initial bpool size configured in queues setup is
 538          * smaller than minimum size add more buffers
 539          */
 540         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
 541         if (priv->bpool_init_size < def_init_size) {
 542                 int buffs_to_add = def_init_size - priv->bpool_init_size;
 543
 544                 priv->bpool_init_size += buffs_to_add;
 545                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
 546                 if (ret)
 547                         MRVL_LOG(ERR, "Failed to add buffers to bpool");
 548         }
 549
 550         /*
 551          * Calculate the maximum bpool size for refill feature as follows:
 552          * maximum number of descriptors in rx queue multiply by number
 553          * of rx queues plus minimum bpool size.
 554          * In case the bpool size will exceed this value, superfluous buffers
 555          * will be removed
 556          */
 557         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
 558                                 priv->bpool_min_size;
 559
 560         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 561         if (ret) {
 562                 MRVL_LOG(ERR, "Failed to init ppio");
 563                 return ret;
 564         }
 565
 566         /*
 567          * In case there are some some stale uc/mc mac addresses flush them
 568          * here. It cannot be done during mrvl_dev_close() as port information
 569          * is already gone at that point (due to pp2_ppio_deinit() in
 570          * mrvl_dev_stop()).
 571          */
 572         if (!priv->uc_mc_flushed) {
 573                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 574                 if (ret) {
 575                         MRVL_LOG(ERR,
 576                                 "Failed to flush uc/mc filter list");
 577                         goto out;
 578                 }
 579                 priv->uc_mc_flushed = 1;
 580         }
 581
 582         if (!priv->vlan_flushed) {
 583                 ret = pp2_ppio_flush_vlan(priv->ppio);
 584                 if (ret) {
 585                         MRVL_LOG(ERR, "Failed to flush vlan list");
 586                         /*
 587                          * TODO
 588                          * once pp2_ppio_flush_vlan() is supported jump to out
 589                          * goto out;
 590                          */
 591                 }
 592                 priv->vlan_flushed = 1;
 593         }
 594
 595         /* For default QoS config, don't start classifier. */
 596         if (mrvl_qos_cfg  &&
 597             mrvl_qos_cfg->port[dev->data->port_id].use_global_defaults == 0) {
 598                 ret = mrvl_start_qos_mapping(priv);
 599                 if (ret) {
 600                         MRVL_LOG(ERR, "Failed to setup QoS mapping");
 601                         goto out;
 602                 }
 603         }
 604
 605         ret = mrvl_dev_set_link_up(dev);
 606         if (ret) {
 607                 MRVL_LOG(ERR, "Failed to set link up");
 608                 goto out;
 609         }
 610
 611         /* start tx queues */
 612         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 613                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 614
 615                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
 616
 617                 if (!txq->tx_deferred_start)
 618                         continue;
 619
 620                 /*
 621                  * All txqs are started by default. Stop them
 622                  * so that tx_deferred_start works as expected.
 623                  */
 624                 ret = mrvl_tx_queue_stop(dev, i);
 625                 if (ret)
 626                         goto out;
 627         }
 628
 629         mrvl_flow_init(dev);
 630         mrvl_mtr_init(dev);
 631
 632         return 0;
 633 out:
 634         MRVL_LOG(ERR, "Failed to start device");
 635         pp2_ppio_deinit(priv->ppio);
 636         return ret;
 637 }
 638
 639 /**
 640  * Flush receive queues.
 641  *
 642  * @param dev
 643  *   Pointer to Ethernet device structure.
 644  */
 645 static void
 646 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 647 {
 648         int i;
 649
 650         MRVL_LOG(INFO, "Flushing rx queues");
 651         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 652                 int ret, num;
 653
 654                 do {
 655                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 656                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 657
 658                         num = MRVL_PP2_RXD_MAX;
 659                         ret = pp2_ppio_recv(q->priv->ppio,
 660                                             q->priv->rxq_map[q->queue_id].tc,
 661                                             q->priv->rxq_map[q->queue_id].inq,
 662                                             descs, (uint16_t *)&num);
 663                 } while (ret == 0 && num);
 664         }
 665 }
 666
 667 /**
 668  * Flush transmit shadow queues.
 669  *
 670  * @param dev
 671  *   Pointer to Ethernet device structure.
 672  */
 673 static void
 674 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 675 {
 676         int i, j;
 677         struct mrvl_txq *txq;
 678
 679         MRVL_LOG(INFO, "Flushing tx shadow queues");
 680         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 681                 txq = (struct mrvl_txq *)dev->data->tx_queues[i];
 682
 683                 for (j = 0; j < RTE_MAX_LCORE; j++) {
 684                         struct mrvl_shadow_txq *sq;
 685
 686                         if (!hifs[j])
 687                                 continue;
 688
 689                         sq = &txq->shadow_txqs[j];
 690                         mrvl_free_sent_buffers(txq->priv->ppio,
 691                                 hifs[j], j, sq, txq->queue_id, 1);
 692                         while (sq->tail != sq->head) {
 693                                 uint64_t addr = cookie_addr_high |
 694                                         sq->ent[sq->tail].buff.cookie;
 695                                 rte_pktmbuf_free(
 696                                         (struct rte_mbuf *)addr);
 697                                 sq->tail = (sq->tail + 1) &
 698                                             MRVL_PP2_TX_SHADOWQ_MASK;
 699                         }
 700                         memset(sq, 0, sizeof(*sq));
 701                 }
 702         }
 703 }
 704
 705 /**
 706  * Flush hardware bpool (buffer-pool).
 707  *
 708  * @param dev
 709  *   Pointer to Ethernet device structure.
 710  */
 711 static void
 712 mrvl_flush_bpool(struct rte_eth_dev *dev)
 713 {
 714         struct mrvl_priv *priv = dev->data->dev_private;
 715         struct pp2_hif *hif;
 716         uint32_t num;
 717         int ret;
 718         unsigned int core_id = rte_lcore_id();
 719
 720         if (core_id == LCORE_ID_ANY)
 721                 core_id = 0;
 722
 723         hif = mrvl_get_hif(priv, core_id);
 724
 725         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 726         if (ret) {
 727                 MRVL_LOG(ERR, "Failed to get bpool buffers number");
 728                 return;
 729         }
 730
 731         while (num--) {
 732                 struct pp2_buff_inf inf;
 733                 uint64_t addr;
 734
 735                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
 736                 if (ret)
 737                         break;
 738
 739                 addr = cookie_addr_high | inf.cookie;
 740                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 741         }
 742 }
 743
 744 /**
 745  * DPDK callback to stop the device.
 746  *
 747  * @param dev
 748  *   Pointer to Ethernet device structure.
 749  */
 750 static void
 751 mrvl_dev_stop(struct rte_eth_dev *dev)
 752 {
 753         mrvl_dev_set_link_down(dev);
 754 }
 755
 756 /**
 757  * DPDK callback to close the device.
 758  *
 759  * @param dev
 760  *   Pointer to Ethernet device structure.
 761  */
 762 static void
 763 mrvl_dev_close(struct rte_eth_dev *dev)
 764 {
 765         struct mrvl_priv *priv = dev->data->dev_private;
 766         size_t i;
 767
 768         mrvl_flush_rx_queues(dev);
 769         mrvl_flush_tx_shadow_queues(dev);
 770         mrvl_flow_deinit(dev);
 771         mrvl_mtr_deinit(dev);
 772
 773         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 774                 struct pp2_ppio_tc_params *tc_params =
 775                         &priv->ppio_params.inqs_params.tcs_params[i];
 776
 777                 if (tc_params->inqs_params) {
 778                         rte_free(tc_params->inqs_params);
 779                         tc_params->inqs_params = NULL;
 780                 }
 781         }
 782
 783         if (priv->cls_tbl) {
 784                 pp2_cls_tbl_deinit(priv->cls_tbl);
 785                 priv->cls_tbl = NULL;
 786         }
 787
 788         if (priv->qos_tbl) {
 789                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 790                 priv->qos_tbl = NULL;
 791         }
 792
 793         mrvl_flush_bpool(dev);
 794         mrvl_tm_deinit(dev);
 795
 796         if (priv->ppio) {
 797                 pp2_ppio_deinit(priv->ppio);
 798                 priv->ppio = NULL;
 799         }
 800
 801         /* policer must be released after ppio deinitialization */
 802         if (priv->default_policer) {
 803                 pp2_cls_plcr_deinit(priv->default_policer);
 804                 priv->default_policer = NULL;
 805         }
 806 }
 807
 808 /**
 809  * DPDK callback to retrieve physical link information.
 810  *
 811  * @param dev
 812  *   Pointer to Ethernet device structure.
 813  * @param wait_to_complete
 814  *   Wait for request completion (ignored).
 815  *
 816  * @return
 817  *   0 on success, negative error value otherwise.
 818  */
 819 static int
 820 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 821 {
 822         /*
 823          * TODO
 824          * once MUSDK provides necessary API use it here
 825          */
 826         struct mrvl_priv *priv = dev->data->dev_private;
 827         struct ethtool_cmd edata;
 828         struct ifreq req;
 829         int ret, fd, link_up;
 830
 831         if (!priv->ppio)
 832                 return -EPERM;
 833
 834         edata.cmd = ETHTOOL_GSET;
 835
 836         strcpy(req.ifr_name, dev->data->name);
 837         req.ifr_data = (void *)&edata;
 838
 839         fd = socket(AF_INET, SOCK_DGRAM, 0);
 840         if (fd == -1)
 841                 return -EFAULT;
 842
 843         ret = ioctl(fd, SIOCETHTOOL, &req);
 844         if (ret == -1) {
 845                 close(fd);
 846                 return -EFAULT;
 847         }
 848
 849         close(fd);
 850
 851         switch (ethtool_cmd_speed(&edata)) {
 852         case SPEED_10:
 853                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 854                 break;
 855         case SPEED_100:
 856                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 857                 break;
 858         case SPEED_1000:
 859                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 860                 break;
 861         case SPEED_10000:
 862                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 863                 break;
 864         default:
 865                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 866         }
 867
 868         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 869                                                          ETH_LINK_HALF_DUPLEX;
 870         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 871                                                            ETH_LINK_FIXED;
 872         pp2_ppio_get_link_state(priv->ppio, &link_up);
 873         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
 874
 875         return 0;
 876 }
 877
 878 /**
 879  * DPDK callback to enable promiscuous mode.
 880  *
 881  * @param dev
 882  *   Pointer to Ethernet device structure.
 883  */
 884 static void
 885 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
 886 {
 887         struct mrvl_priv *priv = dev->data->dev_private;
 888         int ret;
 889
 890         if (!priv->ppio)
 891                 return;
 892
 893         if (priv->isolated)
 894                 return;
 895
 896         ret = pp2_ppio_set_promisc(priv->ppio, 1);
 897         if (ret)
 898                 MRVL_LOG(ERR, "Failed to enable promiscuous mode");
 899 }
 900
 901 /**
 902  * DPDK callback to enable allmulti mode.
 903  *
 904  * @param dev
 905  *   Pointer to Ethernet device structure.
 906  */
 907 static void
 908 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
 909 {
 910         struct mrvl_priv *priv = dev->data->dev_private;
 911         int ret;
 912
 913         if (!priv->ppio)
 914                 return;
 915
 916         if (priv->isolated)
 917                 return;
 918
 919         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
 920         if (ret)
 921                 MRVL_LOG(ERR, "Failed enable all-multicast mode");
 922 }
 923
 924 /**
 925  * DPDK callback to disable promiscuous mode.
 926  *
 927  * @param dev
 928  *   Pointer to Ethernet device structure.
 929  */
 930 static void
 931 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
 932 {
 933         struct mrvl_priv *priv = dev->data->dev_private;
 934         int ret;
 935
 936         if (!priv->ppio)
 937                 return;
 938
 939         ret = pp2_ppio_set_promisc(priv->ppio, 0);
 940         if (ret)
 941                 MRVL_LOG(ERR, "Failed to disable promiscuous mode");
 942 }
 943
 944 /**
 945  * DPDK callback to disable allmulticast mode.
 946  *
 947  * @param dev
 948  *   Pointer to Ethernet device structure.
 949  */
 950 static void
 951 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
 952 {
 953         struct mrvl_priv *priv = dev->data->dev_private;
 954         int ret;
 955
 956         if (!priv->ppio)
 957                 return;
 958
 959         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
 960         if (ret)
 961                 MRVL_LOG(ERR, "Failed to disable all-multicast mode");
 962 }
 963
 964 /**
 965  * DPDK callback to remove a MAC address.
 966  *
 967  * @param dev
 968  *   Pointer to Ethernet device structure.
 969  * @param index
 970  *   MAC address index.
 971  */
 972 static void
 973 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 974 {
 975         struct mrvl_priv *priv = dev->data->dev_private;
 976         char buf[ETHER_ADDR_FMT_SIZE];
 977         int ret;
 978
 979         if (!priv->ppio)
 980                 return;
 981
 982         if (priv->isolated)
 983                 return;
 984
 985         ret = pp2_ppio_remove_mac_addr(priv->ppio,
 986                                        dev->data->mac_addrs[index].addr_bytes);
 987         if (ret) {
 988                 ether_format_addr(buf, sizeof(buf),
 989                                   &dev->data->mac_addrs[index]);
 990                 MRVL_LOG(ERR, "Failed to remove mac %s", buf);
 991         }
 992 }
 993
 994 /**
 995  * DPDK callback to add a MAC address.
 996  *
 997  * @param dev
 998  *   Pointer to Ethernet device structure.
 999  * @param mac_addr
1000  *   MAC address to register.
1001  * @param index
1002  *   MAC address index.
1003  * @param vmdq
1004  *   VMDq pool index to associate address with (unused).
1005  *
1006  * @return
1007  *   0 on success, negative error value otherwise.
1008  */
1009 static int
1010 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
1011                   uint32_t index, uint32_t vmdq __rte_unused)
1012 {
1013         struct mrvl_priv *priv = dev->data->dev_private;
1014         char buf[ETHER_ADDR_FMT_SIZE];
1015         int ret;
1016
1017         if (priv->isolated)
1018                 return -ENOTSUP;
1019
1020         if (index == 0)
1021                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
1022                 return -1;
1023
1024         if (!priv->ppio)
1025                 return 0;
1026
1027         /*
1028          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
1029          * parameter uc_filter_max. Maximum number of mc addresses is then
1030          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
1031          * 21 respectively.
1032          *
1033          * If more than uc_filter_max uc addresses were added to filter list
1034          * then NIC will switch to promiscuous mode automatically.
1035          *
1036          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
1037          * were added to filter list then NIC will switch to all-multicast mode
1038          * automatically.
1039          */
1040         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
1041         if (ret) {
1042                 ether_format_addr(buf, sizeof(buf), mac_addr);
1043                 MRVL_LOG(ERR, "Failed to add mac %s", buf);
1044                 return -1;
1045         }
1046
1047         return 0;
1048 }
1049
1050 /**
1051  * DPDK callback to set the primary MAC address.
1052  *
1053  * @param dev
1054  *   Pointer to Ethernet device structure.
1055  * @param mac_addr
1056  *   MAC address to register.
1057  *
1058  * @return
1059  *   0 on success, negative error value otherwise.
1060  */
1061 static int
1062 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1063 {
1064         struct mrvl_priv *priv = dev->data->dev_private;
1065         int ret;
1066
1067         if (!priv->ppio)
1068                 return 0;
1069
1070         if (priv->isolated)
1071                 return -ENOTSUP;
1072
1073         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
1074         if (ret) {
1075                 char buf[ETHER_ADDR_FMT_SIZE];
1076                 ether_format_addr(buf, sizeof(buf), mac_addr);
1077                 MRVL_LOG(ERR, "Failed to set mac to %s", buf);
1078         }
1079
1080         return ret;
1081 }
1082
1083 /**
1084  * DPDK callback to get device statistics.
1085  *
1086  * @param dev
1087  *   Pointer to Ethernet device structure.
1088  * @param stats
1089  *   Stats structure output buffer.
1090  *
1091  * @return
1092  *   0 on success, negative error value otherwise.
1093  */
1094 static int
1095 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1096 {
1097         struct mrvl_priv *priv = dev->data->dev_private;
1098         struct pp2_ppio_statistics ppio_stats;
1099         uint64_t drop_mac = 0;
1100         unsigned int i, idx, ret;
1101
1102         if (!priv->ppio)
1103                 return -EPERM;
1104
1105         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1106                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1107                 struct pp2_ppio_inq_statistics rx_stats;
1108
1109                 if (!rxq)
1110                         continue;
1111
1112                 idx = rxq->queue_id;
1113                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1114                         MRVL_LOG(ERR,
1115                                 "rx queue %d stats out of range (0 - %d)",
1116                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1117                         continue;
1118                 }
1119
1120                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
1121                                                   priv->rxq_map[idx].tc,
1122                                                   priv->rxq_map[idx].inq,
1123                                                   &rx_stats, 0);
1124                 if (unlikely(ret)) {
1125                         MRVL_LOG(ERR,
1126                                 "Failed to update rx queue %d stats", idx);
1127                         break;
1128                 }
1129
1130                 stats->q_ibytes[idx] = rxq->bytes_recv;
1131                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
1132                 stats->q_errors[idx] = rx_stats.drop_early +
1133                                        rx_stats.drop_fullq +
1134                                        rx_stats.drop_bm +
1135                                        rxq->drop_mac;
1136                 stats->ibytes += rxq->bytes_recv;
1137                 drop_mac += rxq->drop_mac;
1138         }
1139
1140         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1141                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1142                 struct pp2_ppio_outq_statistics tx_stats;
1143
1144                 if (!txq)
1145                         continue;
1146
1147                 idx = txq->queue_id;
1148                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1149                         MRVL_LOG(ERR,
1150                                 "tx queue %d stats out of range (0 - %d)",
1151                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1152                 }
1153
1154                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1155                                                    &tx_stats, 0);
1156                 if (unlikely(ret)) {
1157                         MRVL_LOG(ERR,
1158                                 "Failed to update tx queue %d stats", idx);
1159                         break;
1160                 }
1161
1162                 stats->q_opackets[idx] = tx_stats.deq_desc;
1163                 stats->q_obytes[idx] = txq->bytes_sent;
1164                 stats->obytes += txq->bytes_sent;
1165         }
1166
1167         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1168         if (unlikely(ret)) {
1169                 MRVL_LOG(ERR, "Failed to update port statistics");
1170                 return ret;
1171         }
1172
1173         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1174         stats->opackets += ppio_stats.tx_packets;
1175         stats->imissed += ppio_stats.rx_fullq_dropped +
1176                           ppio_stats.rx_bm_dropped +
1177                           ppio_stats.rx_early_dropped +
1178                           ppio_stats.rx_fifo_dropped +
1179                           ppio_stats.rx_cls_dropped;
1180         stats->ierrors = drop_mac;
1181
1182         return 0;
1183 }
1184
1185 /**
1186  * DPDK callback to clear device statistics.
1187  *
1188  * @param dev
1189  *   Pointer to Ethernet device structure.
1190  */
1191 static void
1192 mrvl_stats_reset(struct rte_eth_dev *dev)
1193 {
1194         struct mrvl_priv *priv = dev->data->dev_private;
1195         int i;
1196
1197         if (!priv->ppio)
1198                 return;
1199
1200         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1201                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1202
1203                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1204                                             priv->rxq_map[i].inq, NULL, 1);
1205                 rxq->bytes_recv = 0;
1206                 rxq->drop_mac = 0;
1207         }
1208
1209         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1210                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1211
1212                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1213                 txq->bytes_sent = 0;
1214         }
1215
1216         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1217 }
1218
1219 /**
1220  * DPDK callback to get extended statistics.
1221  *
1222  * @param dev
1223  *   Pointer to Ethernet device structure.
1224  * @param stats
1225  *   Pointer to xstats table.
1226  * @param n
1227  *   Number of entries in xstats table.
1228  * @return
1229  *   Negative value on error, number of read xstats otherwise.
1230  */
1231 static int
1232 mrvl_xstats_get(struct rte_eth_dev *dev,
1233                 struct rte_eth_xstat *stats, unsigned int n)
1234 {
1235         struct mrvl_priv *priv = dev->data->dev_private;
1236         struct pp2_ppio_statistics ppio_stats;
1237         unsigned int i;
1238
1239         if (!stats)
1240                 return 0;
1241
1242         pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1243         for (i = 0; i < n && i < RTE_DIM(mrvl_xstats_tbl); i++) {
1244                 uint64_t val;
1245
1246                 if (mrvl_xstats_tbl[i].size == sizeof(uint32_t))
1247                         val = *(uint32_t *)((uint8_t *)&ppio_stats +
1248                                             mrvl_xstats_tbl[i].offset);
1249                 else if (mrvl_xstats_tbl[i].size == sizeof(uint64_t))
1250                         val = *(uint64_t *)((uint8_t *)&ppio_stats +
1251                                             mrvl_xstats_tbl[i].offset);
1252                 else
1253                         return -EINVAL;
1254
1255                 stats[i].id = i;
1256                 stats[i].value = val;
1257         }
1258
1259         return n;
1260 }
1261
1262 /**
1263  * DPDK callback to reset extended statistics.
1264  *
1265  * @param dev
1266  *   Pointer to Ethernet device structure.
1267  */
1268 static void
1269 mrvl_xstats_reset(struct rte_eth_dev *dev)
1270 {
1271         mrvl_stats_reset(dev);
1272 }
1273
1274 /**
1275  * DPDK callback to get extended statistics names.
1276  *
1277  * @param dev (unused)
1278  *   Pointer to Ethernet device structure.
1279  * @param xstats_names
1280  *   Pointer to xstats names table.
1281  * @param size
1282  *   Size of the xstats names table.
1283  * @return
1284  *   Number of read names.
1285  */
1286 static int
1287 mrvl_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
1288                       struct rte_eth_xstat_name *xstats_names,
1289                       unsigned int size)
1290 {
1291         unsigned int i;
1292
1293         if (!xstats_names)
1294                 return RTE_DIM(mrvl_xstats_tbl);
1295
1296         for (i = 0; i < size && i < RTE_DIM(mrvl_xstats_tbl); i++)
1297                 snprintf(xstats_names[i].name, RTE_ETH_XSTATS_NAME_SIZE, "%s",
1298                          mrvl_xstats_tbl[i].name);
1299
1300         return size;
1301 }
1302
1303 /**
1304  * DPDK callback to get information about the device.
1305  *
1306  * @param dev
1307  *   Pointer to Ethernet device structure (unused).
1308  * @param info
1309  *   Info structure output buffer.
1310  */
1311 static void
1312 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
1313                    struct rte_eth_dev_info *info)
1314 {
1315         info->speed_capa = ETH_LINK_SPEED_10M |
1316                            ETH_LINK_SPEED_100M |
1317                            ETH_LINK_SPEED_1G |
1318                            ETH_LINK_SPEED_10G;
1319
1320         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1321         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1322         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1323
1324         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1325         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1326         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1327
1328         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1329         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1330         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1331
1332         info->rx_offload_capa = MRVL_RX_OFFLOADS;
1333         info->rx_queue_offload_capa = MRVL_RX_OFFLOADS;
1334
1335         info->tx_offload_capa = MRVL_TX_OFFLOADS;
1336         info->tx_queue_offload_capa = MRVL_TX_OFFLOADS;
1337
1338         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1339                                        ETH_RSS_NONFRAG_IPV4_TCP |
1340                                        ETH_RSS_NONFRAG_IPV4_UDP;
1341
1342         /* By default packets are dropped if no descriptors are available */
1343         info->default_rxconf.rx_drop_en = 1;
1344
1345         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1346 }
1347
1348 /**
1349  * Return supported packet types.
1350  *
1351  * @param dev
1352  *   Pointer to Ethernet device structure (unused).
1353  *
1354  * @return
1355  *   Const pointer to the table with supported packet types.
1356  */
1357 static const uint32_t *
1358 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1359 {
1360         static const uint32_t ptypes[] = {
1361                 RTE_PTYPE_L2_ETHER,
1362                 RTE_PTYPE_L2_ETHER_VLAN,
1363                 RTE_PTYPE_L2_ETHER_QINQ,
1364                 RTE_PTYPE_L3_IPV4,
1365                 RTE_PTYPE_L3_IPV4_EXT,
1366                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1367                 RTE_PTYPE_L3_IPV6,
1368                 RTE_PTYPE_L3_IPV6_EXT,
1369                 RTE_PTYPE_L2_ETHER_ARP,
1370                 RTE_PTYPE_L4_TCP,
1371                 RTE_PTYPE_L4_UDP
1372         };
1373
1374         return ptypes;
1375 }
1376
1377 /**
1378  * DPDK callback to get information about specific receive queue.
1379  *
1380  * @param dev
1381  *   Pointer to Ethernet device structure.
1382  * @param rx_queue_id
1383  *   Receive queue index.
1384  * @param qinfo
1385  *   Receive queue information structure.
1386  */
1387 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1388                               struct rte_eth_rxq_info *qinfo)
1389 {
1390         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1391         struct mrvl_priv *priv = dev->data->dev_private;
1392         int inq = priv->rxq_map[rx_queue_id].inq;
1393         int tc = priv->rxq_map[rx_queue_id].tc;
1394         struct pp2_ppio_tc_params *tc_params =
1395                 &priv->ppio_params.inqs_params.tcs_params[tc];
1396
1397         qinfo->mp = q->mp;
1398         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1399 }
1400
1401 /**
1402  * DPDK callback to get information about specific transmit queue.
1403  *
1404  * @param dev
1405  *   Pointer to Ethernet device structure.
1406  * @param tx_queue_id
1407  *   Transmit queue index.
1408  * @param qinfo
1409  *   Transmit queue information structure.
1410  */
1411 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1412                               struct rte_eth_txq_info *qinfo)
1413 {
1414         struct mrvl_priv *priv = dev->data->dev_private;
1415         struct mrvl_txq *txq = dev->data->tx_queues[tx_queue_id];
1416
1417         qinfo->nb_desc =
1418                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1419         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1420 }
1421
1422 /**
1423  * DPDK callback to Configure a VLAN filter.
1424  *
1425  * @param dev
1426  *   Pointer to Ethernet device structure.
1427  * @param vlan_id
1428  *   VLAN ID to filter.
1429  * @param on
1430  *   Toggle filter.
1431  *
1432  * @return
1433  *   0 on success, negative error value otherwise.
1434  */
1435 static int
1436 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1437 {
1438         struct mrvl_priv *priv = dev->data->dev_private;
1439
1440         if (!priv->ppio)
1441                 return -EPERM;
1442
1443         if (priv->isolated)
1444                 return -ENOTSUP;
1445
1446         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1447                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1448 }
1449
1450 /**
1451  * Release buffers to hardware bpool (buffer-pool)
1452  *
1453  * @param rxq
1454  *   Receive queue pointer.
1455  * @param num
1456  *   Number of buffers to release to bpool.
1457  *
1458  * @return
1459  *   0 on success, negative error value otherwise.
1460  */
1461 static int
1462 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1463 {
1464         struct buff_release_entry entries[MRVL_PP2_RXD_MAX];
1465         struct rte_mbuf *mbufs[MRVL_PP2_RXD_MAX];
1466         int i, ret;
1467         unsigned int core_id;
1468         struct pp2_hif *hif;
1469         struct pp2_bpool *bpool;
1470
1471         core_id = rte_lcore_id();
1472         if (core_id == LCORE_ID_ANY)
1473                 core_id = 0;
1474
1475         hif = mrvl_get_hif(rxq->priv, core_id);
1476         if (!hif)
1477                 return -1;
1478
1479         bpool = rxq->priv->bpool;
1480
1481         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1482         if (ret)
1483                 return ret;
1484
1485         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1486                 cookie_addr_high =
1487                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1488
1489         for (i = 0; i < num; i++) {
1490                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1491                         != cookie_addr_high) {
1492                         MRVL_LOG(ERR,
1493                                 "mbuf virtual addr high 0x%lx out of range",
1494                                 (uint64_t)mbufs[i] >> 32);
1495                         goto out;
1496                 }
1497
1498                 entries[i].buff.addr =
1499                         rte_mbuf_data_iova_default(mbufs[i]);
1500                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
1501                 entries[i].bpool = bpool;
1502         }
1503
1504         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1505         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1506
1507         if (i != num)
1508                 goto out;
1509
1510         return 0;
1511 out:
1512         for (; i < num; i++)
1513                 rte_pktmbuf_free(mbufs[i]);
1514
1515         return -1;
1516 }
1517
1518 /**
1519  * DPDK callback to configure the receive queue.
1520  *
1521  * @param dev
1522  *   Pointer to Ethernet device structure.
1523  * @param idx
1524  *   RX queue index.
1525  * @param desc
1526  *   Number of descriptors to configure in queue.
1527  * @param socket
1528  *   NUMA socket on which memory must be allocated.
1529  * @param conf
1530  *   Thresholds parameters.
1531  * @param mp
1532  *   Memory pool for buffer allocations.
1533  *
1534  * @return
1535  *   0 on success, negative error value otherwise.
1536  */
1537 static int
1538 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1539                     unsigned int socket,
1540                     const struct rte_eth_rxconf *conf,
1541                     struct rte_mempool *mp)
1542 {
1543         struct mrvl_priv *priv = dev->data->dev_private;
1544         struct mrvl_rxq *rxq;
1545         uint32_t min_size,
1546                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1547         int ret, tc, inq;
1548         uint64_t offloads;
1549
1550         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1551
1552         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1553                 /*
1554                  * Unknown TC mapping, mapping will not have a correct queue.
1555                  */
1556                 MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu",
1557                         idx, priv->ppio_id);
1558                 return -EFAULT;
1559         }
1560
1561         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
1562                    MRVL_PKT_EFFEC_OFFS;
1563         if (min_size < max_rx_pkt_len) {
1564                 MRVL_LOG(ERR,
1565                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.",
1566                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
1567                         MRVL_PKT_EFFEC_OFFS,
1568                         max_rx_pkt_len);
1569                 return -EINVAL;
1570         }
1571
1572         if (dev->data->rx_queues[idx]) {
1573                 rte_free(dev->data->rx_queues[idx]);
1574                 dev->data->rx_queues[idx] = NULL;
1575         }
1576
1577         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1578         if (!rxq)
1579                 return -ENOMEM;
1580
1581         rxq->priv = priv;
1582         rxq->mp = mp;
1583         rxq->cksum_enabled = offloads & DEV_RX_OFFLOAD_IPV4_CKSUM;
1584         rxq->queue_id = idx;
1585         rxq->port_id = dev->data->port_id;
1586         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1587
1588         tc = priv->rxq_map[rxq->queue_id].tc,
1589         inq = priv->rxq_map[rxq->queue_id].inq;
1590         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1591                 desc;
1592
1593         ret = mrvl_fill_bpool(rxq, desc);
1594         if (ret) {
1595                 rte_free(rxq);
1596                 return ret;
1597         }
1598
1599         priv->bpool_init_size += desc;
1600
1601         dev->data->rx_queues[idx] = rxq;
1602
1603         return 0;
1604 }
1605
1606 /**
1607  * DPDK callback to release the receive queue.
1608  *
1609  * @param rxq
1610  *   Generic receive queue pointer.
1611  */
1612 static void
1613 mrvl_rx_queue_release(void *rxq)
1614 {
1615         struct mrvl_rxq *q = rxq;
1616         struct pp2_ppio_tc_params *tc_params;
1617         int i, num, tc, inq;
1618         struct pp2_hif *hif;
1619         unsigned int core_id = rte_lcore_id();
1620
1621         if (core_id == LCORE_ID_ANY)
1622                 core_id = 0;
1623
1624         if (!q)
1625                 return;
1626
1627         hif = mrvl_get_hif(q->priv, core_id);
1628
1629         if (!hif)
1630                 return;
1631
1632         tc = q->priv->rxq_map[q->queue_id].tc;
1633         inq = q->priv->rxq_map[q->queue_id].inq;
1634         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1635         num = tc_params->inqs_params[inq].size;
1636         for (i = 0; i < num; i++) {
1637                 struct pp2_buff_inf inf;
1638                 uint64_t addr;
1639
1640                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
1641                 addr = cookie_addr_high | inf.cookie;
1642                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1643         }
1644
1645         rte_free(q);
1646 }
1647
1648 /**
1649  * DPDK callback to configure the transmit queue.
1650  *
1651  * @param dev
1652  *   Pointer to Ethernet device structure.
1653  * @param idx
1654  *   Transmit queue index.
1655  * @param desc
1656  *   Number of descriptors to configure in the queue.
1657  * @param socket
1658  *   NUMA socket on which memory must be allocated.
1659  * @param conf
1660  *   Tx queue configuration parameters.
1661  *
1662  * @return
1663  *   0 on success, negative error value otherwise.
1664  */
1665 static int
1666 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1667                     unsigned int socket,
1668                     const struct rte_eth_txconf *conf)
1669 {
1670         struct mrvl_priv *priv = dev->data->dev_private;
1671         struct mrvl_txq *txq;
1672
1673         if (dev->data->tx_queues[idx]) {
1674                 rte_free(dev->data->tx_queues[idx]);
1675                 dev->data->tx_queues[idx] = NULL;
1676         }
1677
1678         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1679         if (!txq)
1680                 return -ENOMEM;
1681
1682         txq->priv = priv;
1683         txq->queue_id = idx;
1684         txq->port_id = dev->data->port_id;
1685         txq->tx_deferred_start = conf->tx_deferred_start;
1686         dev->data->tx_queues[idx] = txq;
1687
1688         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1689
1690         return 0;
1691 }
1692
1693 /**
1694  * DPDK callback to release the transmit queue.
1695  *
1696  * @param txq
1697  *   Generic transmit queue pointer.
1698  */
1699 static void
1700 mrvl_tx_queue_release(void *txq)
1701 {
1702         struct mrvl_txq *q = txq;
1703
1704         if (!q)
1705                 return;
1706
1707         rte_free(q);
1708 }
1709
1710 /**
1711  * DPDK callback to get flow control configuration.
1712  *
1713  * @param dev
1714  *  Pointer to Ethernet device structure.
1715  * @param fc_conf
1716  *  Pointer to the flow control configuration.
1717  *
1718  * @return
1719  *  0 on success, negative error value otherwise.
1720  */
1721 static int
1722 mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1723 {
1724         struct mrvl_priv *priv = dev->data->dev_private;
1725         int ret, en;
1726
1727         if (!priv)
1728                 return -EPERM;
1729
1730         ret = pp2_ppio_get_rx_pause(priv->ppio, &en);
1731         if (ret) {
1732                 MRVL_LOG(ERR, "Failed to read rx pause state");
1733                 return ret;
1734         }
1735
1736         fc_conf->mode = en ? RTE_FC_RX_PAUSE : RTE_FC_NONE;
1737
1738         return 0;
1739 }
1740
1741 /**
1742  * DPDK callback to set flow control configuration.
1743  *
1744  * @param dev
1745  *  Pointer to Ethernet device structure.
1746  * @param fc_conf
1747  *  Pointer to the flow control configuration.
1748  *
1749  * @return
1750  *  0 on success, negative error value otherwise.
1751  */
1752 static int
1753 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1754 {
1755         struct mrvl_priv *priv = dev->data->dev_private;
1756
1757         if (!priv)
1758                 return -EPERM;
1759
1760         if (fc_conf->high_water ||
1761             fc_conf->low_water ||
1762             fc_conf->pause_time ||
1763             fc_conf->mac_ctrl_frame_fwd ||
1764             fc_conf->autoneg) {
1765                 MRVL_LOG(ERR, "Flowctrl parameter is not supported");
1766
1767                 return -EINVAL;
1768         }
1769
1770         if (fc_conf->mode == RTE_FC_NONE ||
1771             fc_conf->mode == RTE_FC_RX_PAUSE) {
1772                 int ret, en;
1773
1774                 en = fc_conf->mode == RTE_FC_NONE ? 0 : 1;
1775                 ret = pp2_ppio_set_rx_pause(priv->ppio, en);
1776                 if (ret)
1777                         MRVL_LOG(ERR,
1778                                 "Failed to change flowctrl on RX side");
1779
1780                 return ret;
1781         }
1782
1783         return 0;
1784 }
1785
1786 /**
1787  * Update RSS hash configuration
1788  *
1789  * @param dev
1790  *   Pointer to Ethernet device structure.
1791  * @param rss_conf
1792  *   Pointer to RSS configuration.
1793  *
1794  * @return
1795  *   0 on success, negative error value otherwise.
1796  */
1797 static int
1798 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1799                      struct rte_eth_rss_conf *rss_conf)
1800 {
1801         struct mrvl_priv *priv = dev->data->dev_private;
1802
1803         if (priv->isolated)
1804                 return -ENOTSUP;
1805
1806         return mrvl_configure_rss(priv, rss_conf);
1807 }
1808
1809 /**
1810  * DPDK callback to get RSS hash configuration.
1811  *
1812  * @param dev
1813  *   Pointer to Ethernet device structure.
1814  * @rss_conf
1815  *   Pointer to RSS configuration.
1816  *
1817  * @return
1818  *   Always 0.
1819  */
1820 static int
1821 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1822                        struct rte_eth_rss_conf *rss_conf)
1823 {
1824         struct mrvl_priv *priv = dev->data->dev_private;
1825         enum pp2_ppio_hash_type hash_type =
1826                 priv->ppio_params.inqs_params.hash_type;
1827
1828         rss_conf->rss_key = NULL;
1829
1830         if (hash_type == PP2_PPIO_HASH_T_NONE)
1831                 rss_conf->rss_hf = 0;
1832         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1833                 rss_conf->rss_hf = ETH_RSS_IPV4;
1834         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1835                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1836         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1837                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1838
1839         return 0;
1840 }
1841
1842 /**
1843  * DPDK callback to get rte_flow callbacks.
1844  *
1845  * @param dev
1846  *   Pointer to the device structure.
1847  * @param filer_type
1848  *   Flow filter type.
1849  * @param filter_op
1850  *   Flow filter operation.
1851  * @param arg
1852  *   Pointer to pass the flow ops.
1853  *
1854  * @return
1855  *   0 on success, negative error value otherwise.
1856  */
1857 static int
1858 mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
1859                      enum rte_filter_type filter_type,
1860                      enum rte_filter_op filter_op, void *arg)
1861 {
1862         switch (filter_type) {
1863         case RTE_ETH_FILTER_GENERIC:
1864                 if (filter_op != RTE_ETH_FILTER_GET)
1865                         return -EINVAL;
1866                 *(const void **)arg = &mrvl_flow_ops;
1867                 return 0;
1868         default:
1869                 MRVL_LOG(WARNING, "Filter type (%d) not supported",
1870                                 filter_type);
1871                 return -EINVAL;
1872         }
1873 }
1874
1875 /**
1876  * DPDK callback to get rte_mtr callbacks.
1877  *
1878  * @param dev
1879  *   Pointer to the device structure.
1880  * @param ops
1881  *   Pointer to pass the mtr ops.
1882  *
1883  * @return
1884  *   Always 0.
1885  */
1886 static int
1887 mrvl_mtr_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
1888 {
1889         *(const void **)ops = &mrvl_mtr_ops;
1890
1891         return 0;
1892 }
1893
1894 /**
1895  * DPDK callback to get rte_tm callbacks.
1896  *
1897  * @param dev
1898  *   Pointer to the device structure.
1899  * @param ops
1900  *   Pointer to pass the tm ops.
1901  *
1902  * @return
1903  *   Always 0.
1904  */
1905 static int
1906 mrvl_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
1907 {
1908         *(const void **)ops = &mrvl_tm_ops;
1909
1910         return 0;
1911 }
1912
1913 static const struct eth_dev_ops mrvl_ops = {
1914         .dev_configure = mrvl_dev_configure,
1915         .dev_start = mrvl_dev_start,
1916         .dev_stop = mrvl_dev_stop,
1917         .dev_set_link_up = mrvl_dev_set_link_up,
1918         .dev_set_link_down = mrvl_dev_set_link_down,
1919         .dev_close = mrvl_dev_close,
1920         .link_update = mrvl_link_update,
1921         .promiscuous_enable = mrvl_promiscuous_enable,
1922         .allmulticast_enable = mrvl_allmulticast_enable,
1923         .promiscuous_disable = mrvl_promiscuous_disable,
1924         .allmulticast_disable = mrvl_allmulticast_disable,
1925         .mac_addr_remove = mrvl_mac_addr_remove,
1926         .mac_addr_add = mrvl_mac_addr_add,
1927         .mac_addr_set = mrvl_mac_addr_set,
1928         .mtu_set = mrvl_mtu_set,
1929         .stats_get = mrvl_stats_get,
1930         .stats_reset = mrvl_stats_reset,
1931         .xstats_get = mrvl_xstats_get,
1932         .xstats_reset = mrvl_xstats_reset,
1933         .xstats_get_names = mrvl_xstats_get_names,
1934         .dev_infos_get = mrvl_dev_infos_get,
1935         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
1936         .rxq_info_get = mrvl_rxq_info_get,
1937         .txq_info_get = mrvl_txq_info_get,
1938         .vlan_filter_set = mrvl_vlan_filter_set,
1939         .tx_queue_start = mrvl_tx_queue_start,
1940         .tx_queue_stop = mrvl_tx_queue_stop,
1941         .rx_queue_setup = mrvl_rx_queue_setup,
1942         .rx_queue_release = mrvl_rx_queue_release,
1943         .tx_queue_setup = mrvl_tx_queue_setup,
1944         .tx_queue_release = mrvl_tx_queue_release,
1945         .flow_ctrl_get = mrvl_flow_ctrl_get,
1946         .flow_ctrl_set = mrvl_flow_ctrl_set,
1947         .rss_hash_update = mrvl_rss_hash_update,
1948         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1949         .filter_ctrl = mrvl_eth_filter_ctrl,
1950         .mtr_ops_get = mrvl_mtr_ops_get,
1951         .tm_ops_get = mrvl_tm_ops_get,
1952 };
1953
1954 /**
1955  * Return packet type information and l3/l4 offsets.
1956  *
1957  * @param desc
1958  *   Pointer to the received packet descriptor.
1959  * @param l3_offset
1960  *   l3 packet offset.
1961  * @param l4_offset
1962  *   l4 packet offset.
1963  *
1964  * @return
1965  *   Packet type information.
1966  */
1967 static inline uint64_t
1968 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
1969                                     uint8_t *l3_offset, uint8_t *l4_offset)
1970 {
1971         enum pp2_inq_l3_type l3_type;
1972         enum pp2_inq_l4_type l4_type;
1973         enum pp2_inq_vlan_tag vlan_tag;
1974         uint64_t packet_type;
1975
1976         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
1977         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
1978         pp2_ppio_inq_desc_get_vlan_tag(desc, &vlan_tag);
1979
1980         packet_type = RTE_PTYPE_L2_ETHER;
1981
1982         switch (vlan_tag) {
1983         case PP2_INQ_VLAN_TAG_SINGLE:
1984                 packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
1985                 break;
1986         case PP2_INQ_VLAN_TAG_DOUBLE:
1987         case PP2_INQ_VLAN_TAG_TRIPLE:
1988                 packet_type |= RTE_PTYPE_L2_ETHER_QINQ;
1989                 break;
1990         default:
1991                 break;
1992         }
1993
1994         switch (l3_type) {
1995         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
1996                 packet_type |= RTE_PTYPE_L3_IPV4;
1997                 break;
1998         case PP2_INQ_L3_TYPE_IPV4_OK:
1999                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
2000                 break;
2001         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
2002                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
2003                 break;
2004         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
2005                 packet_type |= RTE_PTYPE_L3_IPV6;
2006                 break;
2007         case PP2_INQ_L3_TYPE_IPV6_EXT:
2008                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
2009                 break;
2010         case PP2_INQ_L3_TYPE_ARP:
2011                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
2012                 /*
2013                  * In case of ARP l4_offset is set to wrong value.
2014                  * Set it to proper one so that later on mbuf->l3_len can be
2015                  * calculated subtracting l4_offset and l3_offset.
2016                  */
2017                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
2018                 break;
2019         default:
2020                 MRVL_LOG(DEBUG, "Failed to recognise l3 packet type");
2021                 break;
2022         }
2023
2024         switch (l4_type) {
2025         case PP2_INQ_L4_TYPE_TCP:
2026                 packet_type |= RTE_PTYPE_L4_TCP;
2027                 break;
2028         case PP2_INQ_L4_TYPE_UDP:
2029                 packet_type |= RTE_PTYPE_L4_UDP;
2030                 break;
2031         default:
2032                 MRVL_LOG(DEBUG, "Failed to recognise l4 packet type");
2033                 break;
2034         }
2035
2036         return packet_type;
2037 }
2038
2039 /**
2040  * Get offload information from the received packet descriptor.
2041  *
2042  * @param desc
2043  *   Pointer to the received packet descriptor.
2044  *
2045  * @return
2046  *   Mbuf offload flags.
2047  */
2048 static inline uint64_t
2049 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
2050 {
2051         uint64_t flags;
2052         enum pp2_inq_desc_status status;
2053
2054         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
2055         if (unlikely(status != PP2_DESC_ERR_OK))
2056                 flags = PKT_RX_IP_CKSUM_BAD;
2057         else
2058                 flags = PKT_RX_IP_CKSUM_GOOD;
2059
2060         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
2061         if (unlikely(status != PP2_DESC_ERR_OK))
2062                 flags |= PKT_RX_L4_CKSUM_BAD;
2063         else
2064                 flags |= PKT_RX_L4_CKSUM_GOOD;
2065
2066         return flags;
2067 }
2068
2069 /**
2070  * DPDK callback for receive.
2071  *
2072  * @param rxq
2073  *   Generic pointer to the receive queue.
2074  * @param rx_pkts
2075  *   Array to store received packets.
2076  * @param nb_pkts
2077  *   Maximum number of packets in array.
2078  *
2079  * @return
2080  *   Number of packets successfully received.
2081  */
2082 static uint16_t
2083 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2084 {
2085         struct mrvl_rxq *q = rxq;
2086         struct pp2_ppio_desc descs[nb_pkts];
2087         struct pp2_bpool *bpool;
2088         int i, ret, rx_done = 0;
2089         int num;
2090         struct pp2_hif *hif;
2091         unsigned int core_id = rte_lcore_id();
2092
2093         hif = mrvl_get_hif(q->priv, core_id);
2094
2095         if (unlikely(!q->priv->ppio || !hif))
2096                 return 0;
2097
2098         bpool = q->priv->bpool;
2099
2100         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
2101                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
2102         if (unlikely(ret < 0)) {
2103                 MRVL_LOG(ERR, "Failed to receive packets");
2104                 return 0;
2105         }
2106         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
2107
2108         for (i = 0; i < nb_pkts; i++) {
2109                 struct rte_mbuf *mbuf;
2110                 uint8_t l3_offset, l4_offset;
2111                 enum pp2_inq_desc_status status;
2112                 uint64_t addr;
2113
2114                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2115                         struct pp2_ppio_desc *pref_desc;
2116                         u64 pref_addr;
2117
2118                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
2119                         pref_addr = cookie_addr_high |
2120                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
2121                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
2122                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
2123                 }
2124
2125                 addr = cookie_addr_high |
2126                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
2127                 mbuf = (struct rte_mbuf *)addr;
2128                 rte_pktmbuf_reset(mbuf);
2129
2130                 /* drop packet in case of mac, overrun or resource error */
2131                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
2132                 if (unlikely(status != PP2_DESC_ERR_OK)) {
2133                         struct pp2_buff_inf binf = {
2134                                 .addr = rte_mbuf_data_iova_default(mbuf),
2135                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
2136                         };
2137
2138                         pp2_bpool_put_buff(hif, bpool, &binf);
2139                         mrvl_port_bpool_size
2140                                 [bpool->pp2_id][bpool->id][core_id]++;
2141                         q->drop_mac++;
2142                         continue;
2143                 }
2144
2145                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
2146                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
2147                 mbuf->data_len = mbuf->pkt_len;
2148                 mbuf->port = q->port_id;
2149                 mbuf->packet_type =
2150                         mrvl_desc_to_packet_type_and_offset(&descs[i],
2151                                                             &l3_offset,
2152                                                             &l4_offset);
2153                 mbuf->l2_len = l3_offset;
2154                 mbuf->l3_len = l4_offset - l3_offset;
2155
2156                 if (likely(q->cksum_enabled))
2157                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
2158
2159                 rx_pkts[rx_done++] = mbuf;
2160                 q->bytes_recv += mbuf->pkt_len;
2161         }
2162
2163         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
2164                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
2165
2166                 if (unlikely(num <= q->priv->bpool_min_size ||
2167                              (!rx_done && num < q->priv->bpool_init_size))) {
2168                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
2169                         if (ret)
2170                                 MRVL_LOG(ERR, "Failed to fill bpool");
2171                 } else if (unlikely(num > q->priv->bpool_max_size)) {
2172                         int i;
2173                         int pkt_to_remove = num - q->priv->bpool_init_size;
2174                         struct rte_mbuf *mbuf;
2175                         struct pp2_buff_inf buff;
2176
2177                         MRVL_LOG(DEBUG,
2178                                 "port-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)",
2179                                 bpool->pp2_id, q->priv->ppio->port_id,
2180                                 bpool->id, pkt_to_remove, num,
2181                                 q->priv->bpool_init_size);
2182
2183                         for (i = 0; i < pkt_to_remove; i++) {
2184                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
2185                                 if (ret)
2186                                         break;
2187                                 mbuf = (struct rte_mbuf *)
2188                                         (cookie_addr_high | buff.cookie);
2189                                 rte_pktmbuf_free(mbuf);
2190                         }
2191                         mrvl_port_bpool_size
2192                                 [bpool->pp2_id][bpool->id][core_id] -= i;
2193                 }
2194                 rte_spinlock_unlock(&q->priv->lock);
2195         }
2196
2197         return rx_done;
2198 }
2199
2200 /**
2201  * Prepare offload information.
2202  *
2203  * @param ol_flags
2204  *   Offload flags.
2205  * @param packet_type
2206  *   Packet type bitfield.
2207  * @param l3_type
2208  *   Pointer to the pp2_ouq_l3_type structure.
2209  * @param l4_type
2210  *   Pointer to the pp2_outq_l4_type structure.
2211  * @param gen_l3_cksum
2212  *   Will be set to 1 in case l3 checksum is computed.
2213  * @param l4_cksum
2214  *   Will be set to 1 in case l4 checksum is computed.
2215  *
2216  * @return
2217  *   0 on success, negative error value otherwise.
2218  */
2219 static inline int
2220 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
2221                         enum pp2_outq_l3_type *l3_type,
2222                         enum pp2_outq_l4_type *l4_type,
2223                         int *gen_l3_cksum,
2224                         int *gen_l4_cksum)
2225 {
2226         /*
2227          * Based on ol_flags prepare information
2228          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
2229          * for offloading.
2230          */
2231         if (ol_flags & PKT_TX_IPV4) {
2232                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
2233                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
2234         } else if (ol_flags & PKT_TX_IPV6) {
2235                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
2236                 /* no checksum for ipv6 header */
2237                 *gen_l3_cksum = 0;
2238         } else {
2239                 /* if something different then stop processing */
2240                 return -1;
2241         }
2242
2243         ol_flags &= PKT_TX_L4_MASK;
2244         if ((packet_type & RTE_PTYPE_L4_TCP) &&
2245             ol_flags == PKT_TX_TCP_CKSUM) {
2246                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
2247                 *gen_l4_cksum = 1;
2248         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
2249                    ol_flags == PKT_TX_UDP_CKSUM) {
2250                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
2251                 *gen_l4_cksum = 1;
2252         } else {
2253                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
2254                 /* no checksum for other type */
2255                 *gen_l4_cksum = 0;
2256         }
2257
2258         return 0;
2259 }
2260
2261 /**
2262  * Release already sent buffers to bpool (buffer-pool).
2263  *
2264  * @param ppio
2265  *   Pointer to the port structure.
2266  * @param hif
2267  *   Pointer to the MUSDK hardware interface.
2268  * @param sq
2269  *   Pointer to the shadow queue.
2270  * @param qid
2271  *   Queue id number.
2272  * @param force
2273  *   Force releasing packets.
2274  */
2275 static inline void
2276 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
2277                        unsigned int core_id, struct mrvl_shadow_txq *sq,
2278                        int qid, int force)
2279 {
2280         struct buff_release_entry *entry;
2281         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
2282         int i;
2283
2284         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
2285
2286         sq->num_to_release += nb_done;
2287
2288         if (likely(!force &&
2289                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
2290                 return;
2291
2292         nb_done = sq->num_to_release;
2293         sq->num_to_release = 0;
2294
2295         for (i = 0; i < nb_done; i++) {
2296                 entry = &sq->ent[sq->tail + num];
2297                 if (unlikely(!entry->buff.addr)) {
2298                         MRVL_LOG(ERR,
2299                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!",
2300                                 sq->tail, (u64)entry->buff.cookie,
2301                                 (u64)entry->buff.addr);
2302                         skip_bufs = 1;
2303                         goto skip;
2304                 }
2305
2306                 if (unlikely(!entry->bpool)) {
2307                         struct rte_mbuf *mbuf;
2308
2309                         mbuf = (struct rte_mbuf *)
2310                                (cookie_addr_high | entry->buff.cookie);
2311                         rte_pktmbuf_free(mbuf);
2312                         skip_bufs = 1;
2313                         goto skip;
2314                 }
2315
2316                 mrvl_port_bpool_size
2317                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
2318                 num++;
2319                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
2320                         goto skip;
2321                 continue;
2322 skip:
2323                 if (likely(num))
2324                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2325                 num += skip_bufs;
2326                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2327                 sq->size -= num;
2328                 num = 0;
2329                 skip_bufs = 0;
2330         }
2331
2332         if (likely(num)) {
2333                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2334                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2335                 sq->size -= num;
2336         }
2337 }
2338
2339 /**
2340  * DPDK callback for transmit.
2341  *
2342  * @param txq
2343  *   Generic pointer transmit queue.
2344  * @param tx_pkts
2345  *   Packets to transmit.
2346  * @param nb_pkts
2347  *   Number of packets in array.
2348  *
2349  * @return
2350  *   Number of packets successfully transmitted.
2351  */
2352 static uint16_t
2353 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2354 {
2355         struct mrvl_txq *q = txq;
2356         struct mrvl_shadow_txq *sq;
2357         struct pp2_hif *hif;
2358         struct pp2_ppio_desc descs[nb_pkts];
2359         unsigned int core_id = rte_lcore_id();
2360         int i, ret, bytes_sent = 0;
2361         uint16_t num, sq_free_size;
2362         uint64_t addr;
2363
2364         hif = mrvl_get_hif(q->priv, core_id);
2365         sq = &q->shadow_txqs[core_id];
2366
2367         if (unlikely(!q->priv->ppio || !hif))
2368                 return 0;
2369
2370         if (sq->size)
2371                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2372                                        sq, q->queue_id, 0);
2373
2374         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2375         if (unlikely(nb_pkts > sq_free_size)) {
2376                 MRVL_LOG(DEBUG,
2377                         "No room in shadow queue for %d packets! %d packets will be sent.",
2378                         nb_pkts, sq_free_size);
2379                 nb_pkts = sq_free_size;
2380         }
2381
2382         for (i = 0; i < nb_pkts; i++) {
2383                 struct rte_mbuf *mbuf = tx_pkts[i];
2384                 int gen_l3_cksum, gen_l4_cksum;
2385                 enum pp2_outq_l3_type l3_type;
2386                 enum pp2_outq_l4_type l4_type;
2387
2388                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2389                         struct rte_mbuf *pref_pkt_hdr;
2390
2391                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2392                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2393                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2394                 }
2395
2396                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
2397                 sq->ent[sq->head].buff.addr =
2398                         rte_mbuf_data_iova_default(mbuf);
2399                 sq->ent[sq->head].bpool =
2400                         (unlikely(mbuf->port >= RTE_MAX_ETHPORTS ||
2401                          mbuf->refcnt > 1)) ? NULL :
2402                          mrvl_port_to_bpool_lookup[mbuf->port];
2403                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
2404                 sq->size++;
2405
2406                 pp2_ppio_outq_desc_reset(&descs[i]);
2407                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
2408                                                  rte_pktmbuf_iova(mbuf));
2409                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
2410                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
2411                                                rte_pktmbuf_pkt_len(mbuf));
2412
2413                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2414                 /*
2415                  * in case unsupported ol_flags were passed
2416                  * do not update descriptor offload information
2417                  */
2418                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
2419                                               &l3_type, &l4_type, &gen_l3_cksum,
2420                                               &gen_l4_cksum);
2421                 if (unlikely(ret))
2422                         continue;
2423
2424                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2425                                                   mbuf->l2_len,
2426                                                   mbuf->l2_len + mbuf->l3_len,
2427                                                   gen_l3_cksum, gen_l4_cksum);
2428         }
2429
2430         num = nb_pkts;
2431         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2432         /* number of packets that were not sent */
2433         if (unlikely(num > nb_pkts)) {
2434                 for (i = nb_pkts; i < num; i++) {
2435                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2436                                 MRVL_PP2_TX_SHADOWQ_MASK;
2437                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
2438                         bytes_sent -=
2439                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2440                 }
2441                 sq->size -= num - nb_pkts;
2442         }
2443
2444         q->bytes_sent += bytes_sent;
2445
2446         return nb_pkts;
2447 }
2448
2449 /**
2450  * Initialize packet processor.
2451  *
2452  * @return
2453  *   0 on success, negative error value otherwise.
2454  */
2455 static int
2456 mrvl_init_pp2(void)
2457 {
2458         struct pp2_init_params init_params;
2459
2460         memset(&init_params, 0, sizeof(init_params));
2461         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
2462         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
2463         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
2464
2465         return pp2_init(&init_params);
2466 }
2467
2468 /**
2469  * Deinitialize packet processor.
2470  *
2471  * @return
2472  *   0 on success, negative error value otherwise.
2473  */
2474 static void
2475 mrvl_deinit_pp2(void)
2476 {
2477         pp2_deinit();
2478 }
2479
2480 /**
2481  * Create private device structure.
2482  *
2483  * @param dev_name
2484  *   Pointer to the port name passed in the initialization parameters.
2485  *
2486  * @return
2487  *   Pointer to the newly allocated private device structure.
2488  */
2489 static struct mrvl_priv *
2490 mrvl_priv_create(const char *dev_name)
2491 {
2492         struct pp2_bpool_params bpool_params;
2493         char match[MRVL_MATCH_LEN];
2494         struct mrvl_priv *priv;
2495         int ret, bpool_bit;
2496
2497         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
2498         if (!priv)
2499                 return NULL;
2500
2501         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
2502                                        &priv->pp_id, &priv->ppio_id);
2503         if (ret)
2504                 goto out_free_priv;
2505
2506         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
2507                                      PP2_BPOOL_NUM_POOLS);
2508         if (bpool_bit < 0)
2509                 goto out_free_priv;
2510         priv->bpool_bit = bpool_bit;
2511
2512         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
2513                  priv->bpool_bit);
2514         memset(&bpool_params, 0, sizeof(bpool_params));
2515         bpool_params.match = match;
2516         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
2517         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
2518         if (ret)
2519                 goto out_clear_bpool_bit;
2520
2521         priv->ppio_params.type = PP2_PPIO_T_NIC;
2522         rte_spinlock_init(&priv->lock);
2523
2524         return priv;
2525 out_clear_bpool_bit:
2526         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2527 out_free_priv:
2528         rte_free(priv);
2529         return NULL;
2530 }
2531
2532 /**
2533  * Create device representing Ethernet port.
2534  *
2535  * @param name
2536  *   Pointer to the port's name.
2537  *
2538  * @return
2539  *   0 on success, negative error value otherwise.
2540  */
2541 static int
2542 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
2543 {
2544         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
2545         struct rte_eth_dev *eth_dev;
2546         struct mrvl_priv *priv;
2547         struct ifreq req;
2548
2549         eth_dev = rte_eth_dev_allocate(name);
2550         if (!eth_dev)
2551                 return -ENOMEM;
2552
2553         priv = mrvl_priv_create(name);
2554         if (!priv) {
2555                 ret = -ENOMEM;
2556                 goto out_free_dev;
2557         }
2558
2559         eth_dev->data->mac_addrs =
2560                 rte_zmalloc("mac_addrs",
2561                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2562         if (!eth_dev->data->mac_addrs) {
2563                 MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
2564                 ret = -ENOMEM;
2565                 goto out_free_priv;
2566         }
2567
2568         memset(&req, 0, sizeof(req));
2569         strcpy(req.ifr_name, name);
2570         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2571         if (ret)
2572                 goto out_free_mac;
2573
2574         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2575                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2576
2577         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2578         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
2579         eth_dev->data->kdrv = RTE_KDRV_NONE;
2580         eth_dev->data->dev_private = priv;
2581         eth_dev->device = &vdev->device;
2582         eth_dev->dev_ops = &mrvl_ops;
2583
2584         rte_eth_dev_probing_finish(eth_dev);
2585         return 0;
2586 out_free_mac:
2587         rte_free(eth_dev->data->mac_addrs);
2588 out_free_dev:
2589         rte_eth_dev_release_port(eth_dev);
2590 out_free_priv:
2591         rte_free(priv);
2592
2593         return ret;
2594 }
2595
2596 /**
2597  * Cleanup previously created device representing Ethernet port.
2598  *
2599  * @param name
2600  *   Pointer to the port name.
2601  */
2602 static void
2603 mrvl_eth_dev_destroy(const char *name)
2604 {
2605         struct rte_eth_dev *eth_dev;
2606         struct mrvl_priv *priv;
2607
2608         eth_dev = rte_eth_dev_allocated(name);
2609         if (!eth_dev)
2610                 return;
2611
2612         priv = eth_dev->data->dev_private;
2613         pp2_bpool_deinit(priv->bpool);
2614         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2615         rte_free(priv);
2616         rte_free(eth_dev->data->mac_addrs);
2617         rte_eth_dev_release_port(eth_dev);
2618 }
2619
2620 /**
2621  * Callback used by rte_kvargs_process() during argument parsing.
2622  *
2623  * @param key
2624  *   Pointer to the parsed key (unused).
2625  * @param value
2626  *   Pointer to the parsed value.
2627  * @param extra_args
2628  *   Pointer to the extra arguments which contains address of the
2629  *   table of pointers to parsed interface names.
2630  *
2631  * @return
2632  *   Always 0.
2633  */
2634 static int
2635 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2636                  void *extra_args)
2637 {
2638         struct mrvl_ifnames *ifnames = extra_args;
2639
2640         ifnames->names[ifnames->idx++] = value;
2641
2642         return 0;
2643 }
2644
2645 /**
2646  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2647  */
2648 static void
2649 mrvl_deinit_hifs(void)
2650 {
2651         int i;
2652
2653         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
2654                 if (hifs[i])
2655                         pp2_hif_deinit(hifs[i]);
2656         }
2657         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
2658         memset(hifs, 0, sizeof(hifs));
2659 }
2660
2661 /**
2662  * DPDK callback to register the virtual device.
2663  *
2664  * @param vdev
2665  *   Pointer to the virtual device.
2666  *
2667  * @return
2668  *   0 on success, negative error value otherwise.
2669  */
2670 static int
2671 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2672 {
2673         struct rte_kvargs *kvlist;
2674         struct mrvl_ifnames ifnames;
2675         int ret = -EINVAL;
2676         uint32_t i, ifnum, cfgnum;
2677         const char *params;
2678
2679         params = rte_vdev_device_args(vdev);
2680         if (!params)
2681                 return -EINVAL;
2682
2683         kvlist = rte_kvargs_parse(params, valid_args);
2684         if (!kvlist)
2685                 return -EINVAL;
2686
2687         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2688         if (ifnum > RTE_DIM(ifnames.names))
2689                 goto out_free_kvlist;
2690
2691         ifnames.idx = 0;
2692         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2693                            mrvl_get_ifnames, &ifnames);
2694
2695
2696         /*
2697          * The below system initialization should be done only once,
2698          * on the first provided configuration file
2699          */
2700         if (!mrvl_qos_cfg) {
2701                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2702                 MRVL_LOG(INFO, "Parsing config file!");
2703                 if (cfgnum > 1) {
2704                         MRVL_LOG(ERR, "Cannot handle more than one config file!");
2705                         goto out_free_kvlist;
2706                 } else if (cfgnum == 1) {
2707                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2708                                            mrvl_get_qoscfg, &mrvl_qos_cfg);
2709                 }
2710         }
2711
2712         if (mrvl_dev_num)
2713                 goto init_devices;
2714
2715         MRVL_LOG(INFO, "Perform MUSDK initializations");
2716
2717         ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist);
2718         if (ret)
2719                 goto out_free_kvlist;
2720
2721         ret = mrvl_init_pp2();
2722         if (ret) {
2723                 MRVL_LOG(ERR, "Failed to init PP!");
2724                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2725                 goto out_free_kvlist;
2726         }
2727
2728         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2729         memset(mrvl_port_to_bpool_lookup, 0, sizeof(mrvl_port_to_bpool_lookup));
2730
2731         mrvl_lcore_first = RTE_MAX_LCORE;
2732         mrvl_lcore_last = 0;
2733
2734 init_devices:
2735         for (i = 0; i < ifnum; i++) {
2736                 MRVL_LOG(INFO, "Creating %s", ifnames.names[i]);
2737                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
2738                 if (ret)
2739                         goto out_cleanup;
2740         }
2741         mrvl_dev_num += ifnum;
2742
2743         rte_kvargs_free(kvlist);
2744
2745         return 0;
2746 out_cleanup:
2747         for (; i > 0; i--)
2748                 mrvl_eth_dev_destroy(ifnames.names[i]);
2749
2750         if (mrvl_dev_num == 0) {
2751                 mrvl_deinit_pp2();
2752                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2753         }
2754 out_free_kvlist:
2755         rte_kvargs_free(kvlist);
2756
2757         return ret;
2758 }
2759
2760 /**
2761  * DPDK callback to remove virtual device.
2762  *
2763  * @param vdev
2764  *   Pointer to the removed virtual device.
2765  *
2766  * @return
2767  *   0 on success, negative error value otherwise.
2768  */
2769 static int
2770 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2771 {
2772         int i;
2773         const char *name;
2774
2775         name = rte_vdev_device_name(vdev);
2776         if (!name)
2777                 return -EINVAL;
2778
2779         MRVL_LOG(INFO, "Removing %s", name);
2780
2781         RTE_ETH_FOREACH_DEV(i) { /* FIXME: removing all devices! */
2782                 char ifname[RTE_ETH_NAME_MAX_LEN];
2783
2784                 rte_eth_dev_get_name_by_port(i, ifname);
2785                 mrvl_eth_dev_destroy(ifname);
2786                 mrvl_dev_num--;
2787         }
2788
2789         if (mrvl_dev_num == 0) {
2790                 MRVL_LOG(INFO, "Perform MUSDK deinit");
2791                 mrvl_deinit_hifs();
2792                 mrvl_deinit_pp2();
2793                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2794         }
2795
2796         return 0;
2797 }
2798
2799 static struct rte_vdev_driver pmd_mrvl_drv = {
2800         .probe = rte_pmd_mrvl_probe,
2801         .remove = rte_pmd_mrvl_remove,
2802 };
2803
2804 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv);
2805 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2);
2806
2807 RTE_INIT(mrvl_init_log)
2808 {
2809         mrvl_logtype = rte_log_register("pmd.net.mvpp2");
2810         if (mrvl_logtype >= 0)
2811                 rte_log_set_level(mrvl_logtype, RTE_LOG_NOTICE);
2812 }