drivers/net/mvpp2/mrvl_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Marvell International Ltd.
   3  * Copyright(c) 2017 Semihalf.
   4  * All rights reserved.
   5  */
   6
   7 #include <rte_ethdev_driver.h>
   8 #include <rte_kvargs.h>
   9 #include <rte_log.h>
  10 #include <rte_malloc.h>
  11 #include <rte_bus_vdev.h>
  12
  13 #include <fcntl.h>
  14 #include <linux/ethtool.h>
  15 #include <linux/sockios.h>
  16 #include <net/if.h>
  17 #include <net/if_arp.h>
  18 #include <sys/ioctl.h>
  19 #include <sys/socket.h>
  20 #include <sys/stat.h>
  21 #include <sys/types.h>
  22
  23 #include <rte_mvep_common.h>
  24 #include "mrvl_ethdev.h"
  25 #include "mrvl_qos.h"
  26 #include "mrvl_flow.h"
  27 #include "mrvl_mtr.h"
  28 #include "mrvl_tm.h"
  29
  30 /* bitmask with reserved hifs */
  31 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  32 /* bitmask with reserved bpools */
  33 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  34 /* bitmask with reserved kernel RSS tables */
  35 #define MRVL_MUSDK_RSS_RESERVED 0x01
  36 /* maximum number of available hifs */
  37 #define MRVL_MUSDK_HIFS_MAX 9
  38
  39 /* prefetch shift */
  40 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  41
  42 /* TCAM has 25 entries reserved for uc/mc filter entries */
  43 #define MRVL_MAC_ADDRS_MAX 25
  44 #define MRVL_MATCH_LEN 16
  45 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  46 /* Maximum allowable packet size */
  47 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  48
  49 #define MRVL_IFACE_NAME_ARG "iface"
  50 #define MRVL_CFG_ARG "cfg"
  51
  52 #define MRVL_BURST_SIZE 64
  53
  54 #define MRVL_ARP_LENGTH 28
  55
  56 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  57
  58 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
  59 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
  60
  61 /** Port Rx offload capabilities */
  62 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
  63                           DEV_RX_OFFLOAD_JUMBO_FRAME | \
  64                           DEV_RX_OFFLOAD_CHECKSUM)
  65
  66 /** Port Tx offloads capabilities */
  67 #define MRVL_TX_OFFLOADS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
  68                           DEV_TX_OFFLOAD_UDP_CKSUM | \
  69                           DEV_TX_OFFLOAD_TCP_CKSUM)
  70
  71 static const char * const valid_args[] = {
  72         MRVL_IFACE_NAME_ARG,
  73         MRVL_CFG_ARG,
  74         NULL
  75 };
  76
  77 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
  78 static struct pp2_hif *hifs[RTE_MAX_LCORE];
  79 static int used_bpools[PP2_NUM_PKT_PROC] = {
  80         [0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED
  81 };
  82
  83 static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
  84 static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
  85 static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
  86
  87 int mrvl_logtype;
  88
  89 struct mrvl_ifnames {
  90         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
  91         int idx;
  92 };
  93
  94 /*
  95  * To use buffer harvesting based on loopback port shadow queue structure
  96  * was introduced for buffers information bookkeeping.
  97  *
  98  * Before sending the packet, related buffer information (pp2_buff_inf) is
  99  * stored in shadow queue. After packet is transmitted no longer used
 100  * packet buffer is released back to it's original hardware pool,
 101  * on condition it originated from interface.
 102  * In case it  was generated by application itself i.e: mbuf->port field is
 103  * 0xff then its released to software mempool.
 104  */
 105 struct mrvl_shadow_txq {
 106         int head;           /* write index - used when sending buffers */
 107         int tail;           /* read index - used when releasing buffers */
 108         u16 size;           /* queue occupied size */
 109         u16 num_to_release; /* number of buffers sent, that can be released */
 110         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 111 };
 112
 113 struct mrvl_rxq {
 114         struct mrvl_priv *priv;
 115         struct rte_mempool *mp;
 116         int queue_id;
 117         int port_id;
 118         int cksum_enabled;
 119         uint64_t bytes_recv;
 120         uint64_t drop_mac;
 121 };
 122
 123 struct mrvl_txq {
 124         struct mrvl_priv *priv;
 125         int queue_id;
 126         int port_id;
 127         uint64_t bytes_sent;
 128         struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE];
 129         int tx_deferred_start;
 130 };
 131
 132 static int mrvl_lcore_first;
 133 static int mrvl_lcore_last;
 134 static int mrvl_dev_num;
 135
 136 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
 137 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
 138                         struct pp2_hif *hif, unsigned int core_id,
 139                         struct mrvl_shadow_txq *sq, int qid, int force);
 140
 141 #define MRVL_XSTATS_TBL_ENTRY(name) { \
 142         #name, offsetof(struct pp2_ppio_statistics, name),      \
 143         sizeof(((struct pp2_ppio_statistics *)0)->name)         \
 144 }
 145
 146 /* Table with xstats data */
 147 static struct {
 148         const char *name;
 149         unsigned int offset;
 150         unsigned int size;
 151 } mrvl_xstats_tbl[] = {
 152         MRVL_XSTATS_TBL_ENTRY(rx_bytes),
 153         MRVL_XSTATS_TBL_ENTRY(rx_packets),
 154         MRVL_XSTATS_TBL_ENTRY(rx_unicast_packets),
 155         MRVL_XSTATS_TBL_ENTRY(rx_errors),
 156         MRVL_XSTATS_TBL_ENTRY(rx_fullq_dropped),
 157         MRVL_XSTATS_TBL_ENTRY(rx_bm_dropped),
 158         MRVL_XSTATS_TBL_ENTRY(rx_early_dropped),
 159         MRVL_XSTATS_TBL_ENTRY(rx_fifo_dropped),
 160         MRVL_XSTATS_TBL_ENTRY(rx_cls_dropped),
 161         MRVL_XSTATS_TBL_ENTRY(tx_bytes),
 162         MRVL_XSTATS_TBL_ENTRY(tx_packets),
 163         MRVL_XSTATS_TBL_ENTRY(tx_unicast_packets),
 164         MRVL_XSTATS_TBL_ENTRY(tx_errors)
 165 };
 166
 167 static inline int
 168 mrvl_get_bpool_size(int pp2_id, int pool_id)
 169 {
 170         int i;
 171         int size = 0;
 172
 173         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 174                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 175
 176         return size;
 177 }
 178
 179 static inline int
 180 mrvl_reserve_bit(int *bitmap, int max)
 181 {
 182         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 183
 184         if (n >= max)
 185                 return -1;
 186
 187         *bitmap |= 1 << n;
 188
 189         return n;
 190 }
 191
 192 static int
 193 mrvl_init_hif(int core_id)
 194 {
 195         struct pp2_hif_params params;
 196         char match[MRVL_MATCH_LEN];
 197         int ret;
 198
 199         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
 200         if (ret < 0) {
 201                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 202                 return ret;
 203         }
 204
 205         snprintf(match, sizeof(match), "hif-%d", ret);
 206         memset(&params, 0, sizeof(params));
 207         params.match = match;
 208         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
 209         ret = pp2_hif_init(&params, &hifs[core_id]);
 210         if (ret) {
 211                 MRVL_LOG(ERR, "Failed to initialize hif %d", core_id);
 212                 return ret;
 213         }
 214
 215         return 0;
 216 }
 217
 218 static inline struct pp2_hif*
 219 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
 220 {
 221         int ret;
 222
 223         if (likely(hifs[core_id] != NULL))
 224                 return hifs[core_id];
 225
 226         rte_spinlock_lock(&priv->lock);
 227
 228         ret = mrvl_init_hif(core_id);
 229         if (ret < 0) {
 230                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 231                 goto out;
 232         }
 233
 234         if (core_id < mrvl_lcore_first)
 235                 mrvl_lcore_first = core_id;
 236
 237         if (core_id > mrvl_lcore_last)
 238                 mrvl_lcore_last = core_id;
 239 out:
 240         rte_spinlock_unlock(&priv->lock);
 241
 242         return hifs[core_id];
 243 }
 244
 245 /**
 246  * Configure rss based on dpdk rss configuration.
 247  *
 248  * @param priv
 249  *   Pointer to private structure.
 250  * @param rss_conf
 251  *   Pointer to RSS configuration.
 252  *
 253  * @return
 254  *   0 on success, negative error value otherwise.
 255  */
 256 static int
 257 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 258 {
 259         if (rss_conf->rss_key)
 260                 MRVL_LOG(WARNING, "Changing hash key is not supported");
 261
 262         if (rss_conf->rss_hf == 0) {
 263                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 264         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 265                 priv->ppio_params.inqs_params.hash_type =
 266                         PP2_PPIO_HASH_T_2_TUPLE;
 267         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 268                 priv->ppio_params.inqs_params.hash_type =
 269                         PP2_PPIO_HASH_T_5_TUPLE;
 270                 priv->rss_hf_tcp = 1;
 271         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 272                 priv->ppio_params.inqs_params.hash_type =
 273                         PP2_PPIO_HASH_T_5_TUPLE;
 274                 priv->rss_hf_tcp = 0;
 275         } else {
 276                 return -EINVAL;
 277         }
 278
 279         return 0;
 280 }
 281
 282 /**
 283  * Ethernet device configuration.
 284  *
 285  * Prepare the driver for a given number of TX and RX queues and
 286  * configure RSS.
 287  *
 288  * @param dev
 289  *   Pointer to Ethernet device structure.
 290  *
 291  * @return
 292  *   0 on success, negative error value otherwise.
 293  */
 294 static int
 295 mrvl_dev_configure(struct rte_eth_dev *dev)
 296 {
 297         struct mrvl_priv *priv = dev->data->dev_private;
 298         int ret;
 299
 300         if (priv->ppio) {
 301                 MRVL_LOG(INFO, "Device reconfiguration is not supported");
 302                 return -EINVAL;
 303         }
 304
 305         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 306             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 307                 MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
 308                         dev->data->dev_conf.rxmode.mq_mode);
 309                 return -EINVAL;
 310         }
 311
 312         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 313                 MRVL_LOG(INFO, "Split headers not supported");
 314                 return -EINVAL;
 315         }
 316
 317         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 318                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 319                                  ETHER_HDR_LEN - ETHER_CRC_LEN;
 320
 321         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 322                                   dev->data->nb_rx_queues);
 323         if (ret < 0)
 324                 return ret;
 325
 326         ret = mrvl_configure_txqs(priv, dev->data->port_id,
 327                                   dev->data->nb_tx_queues);
 328         if (ret < 0)
 329                 return ret;
 330
 331         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 332         priv->ppio_params.maintain_stats = 1;
 333         priv->nb_rx_queues = dev->data->nb_rx_queues;
 334
 335         ret = mrvl_tm_init(dev);
 336         if (ret < 0)
 337                 return ret;
 338
 339         if (dev->data->nb_rx_queues == 1 &&
 340             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 341                 MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
 342                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 343
 344                 return 0;
 345         }
 346
 347         return mrvl_configure_rss(priv,
 348                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
 349 }
 350
 351 /**
 352  * DPDK callback to change the MTU.
 353  *
 354  * Setting the MTU affects hardware MRU (packets larger than the MRU
 355  * will be dropped).
 356  *
 357  * @param dev
 358  *   Pointer to Ethernet device structure.
 359  * @param mtu
 360  *   New MTU.
 361  *
 362  * @return
 363  *   0 on success, negative error value otherwise.
 364  */
 365 static int
 366 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 367 {
 368         struct mrvl_priv *priv = dev->data->dev_private;
 369         /* extra MV_MH_SIZE bytes are required for Marvell tag */
 370         uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
 371         int ret;
 372
 373         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
 374                 return -EINVAL;
 375
 376         if (!priv->ppio)
 377                 return 0;
 378
 379         ret = pp2_ppio_set_mru(priv->ppio, mru);
 380         if (ret)
 381                 return ret;
 382
 383         return pp2_ppio_set_mtu(priv->ppio, mtu);
 384 }
 385
 386 /**
 387  * DPDK callback to bring the link up.
 388  *
 389  * @param dev
 390  *   Pointer to Ethernet device structure.
 391  *
 392  * @return
 393  *   0 on success, negative error value otherwise.
 394  */
 395 static int
 396 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 397 {
 398         struct mrvl_priv *priv = dev->data->dev_private;
 399         int ret;
 400
 401         if (!priv->ppio)
 402                 return -EPERM;
 403
 404         ret = pp2_ppio_enable(priv->ppio);
 405         if (ret)
 406                 return ret;
 407
 408         /*
 409          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 410          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 411          * PP2_TRAFFIC_INGRESS_EGRESS.
 412          *
 413          * Set mtu to default DPDK value here.
 414          */
 415         ret = mrvl_mtu_set(dev, dev->data->mtu);
 416         if (ret)
 417                 pp2_ppio_disable(priv->ppio);
 418
 419         return ret;
 420 }
 421
 422 /**
 423  * DPDK callback to bring the link down.
 424  *
 425  * @param dev
 426  *   Pointer to Ethernet device structure.
 427  *
 428  * @return
 429  *   0 on success, negative error value otherwise.
 430  */
 431 static int
 432 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 433 {
 434         struct mrvl_priv *priv = dev->data->dev_private;
 435
 436         if (!priv->ppio)
 437                 return -EPERM;
 438
 439         return pp2_ppio_disable(priv->ppio);
 440 }
 441
 442 /**
 443  * DPDK callback to start tx queue.
 444  *
 445  * @param dev
 446  *   Pointer to Ethernet device structure.
 447  * @param queue_id
 448  *   Transmit queue index.
 449  *
 450  * @return
 451  *   0 on success, negative error value otherwise.
 452  */
 453 static int
 454 mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
 455 {
 456         struct mrvl_priv *priv = dev->data->dev_private;
 457         int ret;
 458
 459         if (!priv)
 460                 return -EPERM;
 461
 462         /* passing 1 enables given tx queue */
 463         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1);
 464         if (ret) {
 465                 MRVL_LOG(ERR, "Failed to start txq %d", queue_id);
 466                 return ret;
 467         }
 468
 469         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 470
 471         return 0;
 472 }
 473
 474 /**
 475  * DPDK callback to stop tx queue.
 476  *
 477  * @param dev
 478  *   Pointer to Ethernet device structure.
 479  * @param queue_id
 480  *   Transmit queue index.
 481  *
 482  * @return
 483  *   0 on success, negative error value otherwise.
 484  */
 485 static int
 486 mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
 487 {
 488         struct mrvl_priv *priv = dev->data->dev_private;
 489         int ret;
 490
 491         if (!priv->ppio)
 492                 return -EPERM;
 493
 494         /* passing 0 disables given tx queue */
 495         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0);
 496         if (ret) {
 497                 MRVL_LOG(ERR, "Failed to stop txq %d", queue_id);
 498                 return ret;
 499         }
 500
 501         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 502
 503         return 0;
 504 }
 505
 506 /**
 507  * DPDK callback to start the device.
 508  *
 509  * @param dev
 510  *   Pointer to Ethernet device structure.
 511  *
 512  * @return
 513  *   0 on success, negative errno value on failure.
 514  */
 515 static int
 516 mrvl_dev_start(struct rte_eth_dev *dev)
 517 {
 518         struct mrvl_priv *priv = dev->data->dev_private;
 519         char match[MRVL_MATCH_LEN];
 520         int ret = 0, i, def_init_size;
 521
 522         if (priv->ppio)
 523                 return mrvl_dev_set_link_up(dev);
 524
 525         snprintf(match, sizeof(match), "ppio-%d:%d",
 526                  priv->pp_id, priv->ppio_id);
 527         priv->ppio_params.match = match;
 528
 529         /*
 530          * Calculate the minimum bpool size for refill feature as follows:
 531          * 2 default burst sizes multiply by number of rx queues.
 532          * If the bpool size will be below this value, new buffers will
 533          * be added to the pool.
 534          */
 535         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 536
 537         /* In case initial bpool size configured in queues setup is
 538          * smaller than minimum size add more buffers
 539          */
 540         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
 541         if (priv->bpool_init_size < def_init_size) {
 542                 int buffs_to_add = def_init_size - priv->bpool_init_size;
 543
 544                 priv->bpool_init_size += buffs_to_add;
 545                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
 546                 if (ret)
 547                         MRVL_LOG(ERR, "Failed to add buffers to bpool");
 548         }
 549
 550         /*
 551          * Calculate the maximum bpool size for refill feature as follows:
 552          * maximum number of descriptors in rx queue multiply by number
 553          * of rx queues plus minimum bpool size.
 554          * In case the bpool size will exceed this value, superfluous buffers
 555          * will be removed
 556          */
 557         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
 558                                 priv->bpool_min_size;
 559
 560         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 561         if (ret) {
 562                 MRVL_LOG(ERR, "Failed to init ppio");
 563                 return ret;
 564         }
 565
 566         /*
 567          * In case there are some some stale uc/mc mac addresses flush them
 568          * here. It cannot be done during mrvl_dev_close() as port information
 569          * is already gone at that point (due to pp2_ppio_deinit() in
 570          * mrvl_dev_stop()).
 571          */
 572         if (!priv->uc_mc_flushed) {
 573                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 574                 if (ret) {
 575                         MRVL_LOG(ERR,
 576                                 "Failed to flush uc/mc filter list");
 577                         goto out;
 578                 }
 579                 priv->uc_mc_flushed = 1;
 580         }
 581
 582         if (!priv->vlan_flushed) {
 583                 ret = pp2_ppio_flush_vlan(priv->ppio);
 584                 if (ret) {
 585                         MRVL_LOG(ERR, "Failed to flush vlan list");
 586                         /*
 587                          * TODO
 588                          * once pp2_ppio_flush_vlan() is supported jump to out
 589                          * goto out;
 590                          */
 591                 }
 592                 priv->vlan_flushed = 1;
 593         }
 594
 595         /* For default QoS config, don't start classifier. */
 596         if (mrvl_qos_cfg) {
 597                 ret = mrvl_start_qos_mapping(priv);
 598                 if (ret) {
 599                         MRVL_LOG(ERR, "Failed to setup QoS mapping");
 600                         goto out;
 601                 }
 602         }
 603
 604         ret = mrvl_dev_set_link_up(dev);
 605         if (ret) {
 606                 MRVL_LOG(ERR, "Failed to set link up");
 607                 goto out;
 608         }
 609
 610         /* start tx queues */
 611         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 612                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 613
 614                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
 615
 616                 if (!txq->tx_deferred_start)
 617                         continue;
 618
 619                 /*
 620                  * All txqs are started by default. Stop them
 621                  * so that tx_deferred_start works as expected.
 622                  */
 623                 ret = mrvl_tx_queue_stop(dev, i);
 624                 if (ret)
 625                         goto out;
 626         }
 627
 628         mrvl_flow_init(dev);
 629         mrvl_mtr_init(dev);
 630
 631         return 0;
 632 out:
 633         MRVL_LOG(ERR, "Failed to start device");
 634         pp2_ppio_deinit(priv->ppio);
 635         return ret;
 636 }
 637
 638 /**
 639  * Flush receive queues.
 640  *
 641  * @param dev
 642  *   Pointer to Ethernet device structure.
 643  */
 644 static void
 645 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 646 {
 647         int i;
 648
 649         MRVL_LOG(INFO, "Flushing rx queues");
 650         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 651                 int ret, num;
 652
 653                 do {
 654                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 655                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 656
 657                         num = MRVL_PP2_RXD_MAX;
 658                         ret = pp2_ppio_recv(q->priv->ppio,
 659                                             q->priv->rxq_map[q->queue_id].tc,
 660                                             q->priv->rxq_map[q->queue_id].inq,
 661                                             descs, (uint16_t *)&num);
 662                 } while (ret == 0 && num);
 663         }
 664 }
 665
 666 /**
 667  * Flush transmit shadow queues.
 668  *
 669  * @param dev
 670  *   Pointer to Ethernet device structure.
 671  */
 672 static void
 673 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 674 {
 675         int i, j;
 676         struct mrvl_txq *txq;
 677
 678         MRVL_LOG(INFO, "Flushing tx shadow queues");
 679         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 680                 txq = (struct mrvl_txq *)dev->data->tx_queues[i];
 681
 682                 for (j = 0; j < RTE_MAX_LCORE; j++) {
 683                         struct mrvl_shadow_txq *sq;
 684
 685                         if (!hifs[j])
 686                                 continue;
 687
 688                         sq = &txq->shadow_txqs[j];
 689                         mrvl_free_sent_buffers(txq->priv->ppio,
 690                                 hifs[j], j, sq, txq->queue_id, 1);
 691                         while (sq->tail != sq->head) {
 692                                 uint64_t addr = cookie_addr_high |
 693                                         sq->ent[sq->tail].buff.cookie;
 694                                 rte_pktmbuf_free(
 695                                         (struct rte_mbuf *)addr);
 696                                 sq->tail = (sq->tail + 1) &
 697                                             MRVL_PP2_TX_SHADOWQ_MASK;
 698                         }
 699                         memset(sq, 0, sizeof(*sq));
 700                 }
 701         }
 702 }
 703
 704 /**
 705  * Flush hardware bpool (buffer-pool).
 706  *
 707  * @param dev
 708  *   Pointer to Ethernet device structure.
 709  */
 710 static void
 711 mrvl_flush_bpool(struct rte_eth_dev *dev)
 712 {
 713         struct mrvl_priv *priv = dev->data->dev_private;
 714         struct pp2_hif *hif;
 715         uint32_t num;
 716         int ret;
 717         unsigned int core_id = rte_lcore_id();
 718
 719         if (core_id == LCORE_ID_ANY)
 720                 core_id = 0;
 721
 722         hif = mrvl_get_hif(priv, core_id);
 723
 724         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 725         if (ret) {
 726                 MRVL_LOG(ERR, "Failed to get bpool buffers number");
 727                 return;
 728         }
 729
 730         while (num--) {
 731                 struct pp2_buff_inf inf;
 732                 uint64_t addr;
 733
 734                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
 735                 if (ret)
 736                         break;
 737
 738                 addr = cookie_addr_high | inf.cookie;
 739                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 740         }
 741 }
 742
 743 /**
 744  * DPDK callback to stop the device.
 745  *
 746  * @param dev
 747  *   Pointer to Ethernet device structure.
 748  */
 749 static void
 750 mrvl_dev_stop(struct rte_eth_dev *dev)
 751 {
 752         mrvl_dev_set_link_down(dev);
 753 }
 754
 755 /**
 756  * DPDK callback to close the device.
 757  *
 758  * @param dev
 759  *   Pointer to Ethernet device structure.
 760  */
 761 static void
 762 mrvl_dev_close(struct rte_eth_dev *dev)
 763 {
 764         struct mrvl_priv *priv = dev->data->dev_private;
 765         size_t i;
 766
 767         mrvl_flush_rx_queues(dev);
 768         mrvl_flush_tx_shadow_queues(dev);
 769         mrvl_flow_deinit(dev);
 770         mrvl_mtr_deinit(dev);
 771
 772         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 773                 struct pp2_ppio_tc_params *tc_params =
 774                         &priv->ppio_params.inqs_params.tcs_params[i];
 775
 776                 if (tc_params->inqs_params) {
 777                         rte_free(tc_params->inqs_params);
 778                         tc_params->inqs_params = NULL;
 779                 }
 780         }
 781
 782         if (priv->cls_tbl) {
 783                 pp2_cls_tbl_deinit(priv->cls_tbl);
 784                 priv->cls_tbl = NULL;
 785         }
 786
 787         if (priv->qos_tbl) {
 788                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 789                 priv->qos_tbl = NULL;
 790         }
 791
 792         mrvl_flush_bpool(dev);
 793         mrvl_tm_deinit(dev);
 794
 795         if (priv->ppio) {
 796                 pp2_ppio_deinit(priv->ppio);
 797                 priv->ppio = NULL;
 798         }
 799
 800         /* policer must be released after ppio deinitialization */
 801         if (priv->default_policer) {
 802                 pp2_cls_plcr_deinit(priv->default_policer);
 803                 priv->default_policer = NULL;
 804         }
 805 }
 806
 807 /**
 808  * DPDK callback to retrieve physical link information.
 809  *
 810  * @param dev
 811  *   Pointer to Ethernet device structure.
 812  * @param wait_to_complete
 813  *   Wait for request completion (ignored).
 814  *
 815  * @return
 816  *   0 on success, negative error value otherwise.
 817  */
 818 static int
 819 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 820 {
 821         /*
 822          * TODO
 823          * once MUSDK provides necessary API use it here
 824          */
 825         struct mrvl_priv *priv = dev->data->dev_private;
 826         struct ethtool_cmd edata;
 827         struct ifreq req;
 828         int ret, fd, link_up;
 829
 830         if (!priv->ppio)
 831                 return -EPERM;
 832
 833         edata.cmd = ETHTOOL_GSET;
 834
 835         strcpy(req.ifr_name, dev->data->name);
 836         req.ifr_data = (void *)&edata;
 837
 838         fd = socket(AF_INET, SOCK_DGRAM, 0);
 839         if (fd == -1)
 840                 return -EFAULT;
 841
 842         ret = ioctl(fd, SIOCETHTOOL, &req);
 843         if (ret == -1) {
 844                 close(fd);
 845                 return -EFAULT;
 846         }
 847
 848         close(fd);
 849
 850         switch (ethtool_cmd_speed(&edata)) {
 851         case SPEED_10:
 852                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 853                 break;
 854         case SPEED_100:
 855                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 856                 break;
 857         case SPEED_1000:
 858                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 859                 break;
 860         case SPEED_10000:
 861                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 862                 break;
 863         default:
 864                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 865         }
 866
 867         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 868                                                          ETH_LINK_HALF_DUPLEX;
 869         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 870                                                            ETH_LINK_FIXED;
 871         pp2_ppio_get_link_state(priv->ppio, &link_up);
 872         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
 873
 874         return 0;
 875 }
 876
 877 /**
 878  * DPDK callback to enable promiscuous mode.
 879  *
 880  * @param dev
 881  *   Pointer to Ethernet device structure.
 882  */
 883 static void
 884 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
 885 {
 886         struct mrvl_priv *priv = dev->data->dev_private;
 887         int ret;
 888
 889         if (!priv->ppio)
 890                 return;
 891
 892         if (priv->isolated)
 893                 return;
 894
 895         ret = pp2_ppio_set_promisc(priv->ppio, 1);
 896         if (ret)
 897                 MRVL_LOG(ERR, "Failed to enable promiscuous mode");
 898 }
 899
 900 /**
 901  * DPDK callback to enable allmulti mode.
 902  *
 903  * @param dev
 904  *   Pointer to Ethernet device structure.
 905  */
 906 static void
 907 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
 908 {
 909         struct mrvl_priv *priv = dev->data->dev_private;
 910         int ret;
 911
 912         if (!priv->ppio)
 913                 return;
 914
 915         if (priv->isolated)
 916                 return;
 917
 918         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
 919         if (ret)
 920                 MRVL_LOG(ERR, "Failed enable all-multicast mode");
 921 }
 922
 923 /**
 924  * DPDK callback to disable promiscuous mode.
 925  *
 926  * @param dev
 927  *   Pointer to Ethernet device structure.
 928  */
 929 static void
 930 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
 931 {
 932         struct mrvl_priv *priv = dev->data->dev_private;
 933         int ret;
 934
 935         if (!priv->ppio)
 936                 return;
 937
 938         ret = pp2_ppio_set_promisc(priv->ppio, 0);
 939         if (ret)
 940                 MRVL_LOG(ERR, "Failed to disable promiscuous mode");
 941 }
 942
 943 /**
 944  * DPDK callback to disable allmulticast mode.
 945  *
 946  * @param dev
 947  *   Pointer to Ethernet device structure.
 948  */
 949 static void
 950 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
 951 {
 952         struct mrvl_priv *priv = dev->data->dev_private;
 953         int ret;
 954
 955         if (!priv->ppio)
 956                 return;
 957
 958         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
 959         if (ret)
 960                 MRVL_LOG(ERR, "Failed to disable all-multicast mode");
 961 }
 962
 963 /**
 964  * DPDK callback to remove a MAC address.
 965  *
 966  * @param dev
 967  *   Pointer to Ethernet device structure.
 968  * @param index
 969  *   MAC address index.
 970  */
 971 static void
 972 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 973 {
 974         struct mrvl_priv *priv = dev->data->dev_private;
 975         char buf[ETHER_ADDR_FMT_SIZE];
 976         int ret;
 977
 978         if (!priv->ppio)
 979                 return;
 980
 981         if (priv->isolated)
 982                 return;
 983
 984         ret = pp2_ppio_remove_mac_addr(priv->ppio,
 985                                        dev->data->mac_addrs[index].addr_bytes);
 986         if (ret) {
 987                 ether_format_addr(buf, sizeof(buf),
 988                                   &dev->data->mac_addrs[index]);
 989                 MRVL_LOG(ERR, "Failed to remove mac %s", buf);
 990         }
 991 }
 992
 993 /**
 994  * DPDK callback to add a MAC address.
 995  *
 996  * @param dev
 997  *   Pointer to Ethernet device structure.
 998  * @param mac_addr
 999  *   MAC address to register.
1000  * @param index
1001  *   MAC address index.
1002  * @param vmdq
1003  *   VMDq pool index to associate address with (unused).
1004  *
1005  * @return
1006  *   0 on success, negative error value otherwise.
1007  */
1008 static int
1009 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
1010                   uint32_t index, uint32_t vmdq __rte_unused)
1011 {
1012         struct mrvl_priv *priv = dev->data->dev_private;
1013         char buf[ETHER_ADDR_FMT_SIZE];
1014         int ret;
1015
1016         if (priv->isolated)
1017                 return -ENOTSUP;
1018
1019         if (index == 0)
1020                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
1021                 return -1;
1022
1023         if (!priv->ppio)
1024                 return 0;
1025
1026         /*
1027          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
1028          * parameter uc_filter_max. Maximum number of mc addresses is then
1029          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
1030          * 21 respectively.
1031          *
1032          * If more than uc_filter_max uc addresses were added to filter list
1033          * then NIC will switch to promiscuous mode automatically.
1034          *
1035          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
1036          * were added to filter list then NIC will switch to all-multicast mode
1037          * automatically.
1038          */
1039         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
1040         if (ret) {
1041                 ether_format_addr(buf, sizeof(buf), mac_addr);
1042                 MRVL_LOG(ERR, "Failed to add mac %s", buf);
1043                 return -1;
1044         }
1045
1046         return 0;
1047 }
1048
1049 /**
1050  * DPDK callback to set the primary MAC address.
1051  *
1052  * @param dev
1053  *   Pointer to Ethernet device structure.
1054  * @param mac_addr
1055  *   MAC address to register.
1056  *
1057  * @return
1058  *   0 on success, negative error value otherwise.
1059  */
1060 static int
1061 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1062 {
1063         struct mrvl_priv *priv = dev->data->dev_private;
1064         int ret;
1065
1066         if (!priv->ppio)
1067                 return 0;
1068
1069         if (priv->isolated)
1070                 return -ENOTSUP;
1071
1072         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
1073         if (ret) {
1074                 char buf[ETHER_ADDR_FMT_SIZE];
1075                 ether_format_addr(buf, sizeof(buf), mac_addr);
1076                 MRVL_LOG(ERR, "Failed to set mac to %s", buf);
1077         }
1078
1079         return ret;
1080 }
1081
1082 /**
1083  * DPDK callback to get device statistics.
1084  *
1085  * @param dev
1086  *   Pointer to Ethernet device structure.
1087  * @param stats
1088  *   Stats structure output buffer.
1089  *
1090  * @return
1091  *   0 on success, negative error value otherwise.
1092  */
1093 static int
1094 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1095 {
1096         struct mrvl_priv *priv = dev->data->dev_private;
1097         struct pp2_ppio_statistics ppio_stats;
1098         uint64_t drop_mac = 0;
1099         unsigned int i, idx, ret;
1100
1101         if (!priv->ppio)
1102                 return -EPERM;
1103
1104         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1105                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1106                 struct pp2_ppio_inq_statistics rx_stats;
1107
1108                 if (!rxq)
1109                         continue;
1110
1111                 idx = rxq->queue_id;
1112                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1113                         MRVL_LOG(ERR,
1114                                 "rx queue %d stats out of range (0 - %d)",
1115                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1116                         continue;
1117                 }
1118
1119                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
1120                                                   priv->rxq_map[idx].tc,
1121                                                   priv->rxq_map[idx].inq,
1122                                                   &rx_stats, 0);
1123                 if (unlikely(ret)) {
1124                         MRVL_LOG(ERR,
1125                                 "Failed to update rx queue %d stats", idx);
1126                         break;
1127                 }
1128
1129                 stats->q_ibytes[idx] = rxq->bytes_recv;
1130                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
1131                 stats->q_errors[idx] = rx_stats.drop_early +
1132                                        rx_stats.drop_fullq +
1133                                        rx_stats.drop_bm +
1134                                        rxq->drop_mac;
1135                 stats->ibytes += rxq->bytes_recv;
1136                 drop_mac += rxq->drop_mac;
1137         }
1138
1139         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1140                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1141                 struct pp2_ppio_outq_statistics tx_stats;
1142
1143                 if (!txq)
1144                         continue;
1145
1146                 idx = txq->queue_id;
1147                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1148                         MRVL_LOG(ERR,
1149                                 "tx queue %d stats out of range (0 - %d)",
1150                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1151                 }
1152
1153                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1154                                                    &tx_stats, 0);
1155                 if (unlikely(ret)) {
1156                         MRVL_LOG(ERR,
1157                                 "Failed to update tx queue %d stats", idx);
1158                         break;
1159                 }
1160
1161                 stats->q_opackets[idx] = tx_stats.deq_desc;
1162                 stats->q_obytes[idx] = txq->bytes_sent;
1163                 stats->obytes += txq->bytes_sent;
1164         }
1165
1166         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1167         if (unlikely(ret)) {
1168                 MRVL_LOG(ERR, "Failed to update port statistics");
1169                 return ret;
1170         }
1171
1172         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1173         stats->opackets += ppio_stats.tx_packets;
1174         stats->imissed += ppio_stats.rx_fullq_dropped +
1175                           ppio_stats.rx_bm_dropped +
1176                           ppio_stats.rx_early_dropped +
1177                           ppio_stats.rx_fifo_dropped +
1178                           ppio_stats.rx_cls_dropped;
1179         stats->ierrors = drop_mac;
1180
1181         return 0;
1182 }
1183
1184 /**
1185  * DPDK callback to clear device statistics.
1186  *
1187  * @param dev
1188  *   Pointer to Ethernet device structure.
1189  */
1190 static void
1191 mrvl_stats_reset(struct rte_eth_dev *dev)
1192 {
1193         struct mrvl_priv *priv = dev->data->dev_private;
1194         int i;
1195
1196         if (!priv->ppio)
1197                 return;
1198
1199         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1200                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1201
1202                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1203                                             priv->rxq_map[i].inq, NULL, 1);
1204                 rxq->bytes_recv = 0;
1205                 rxq->drop_mac = 0;
1206         }
1207
1208         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1209                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1210
1211                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1212                 txq->bytes_sent = 0;
1213         }
1214
1215         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1216 }
1217
1218 /**
1219  * DPDK callback to get extended statistics.
1220  *
1221  * @param dev
1222  *   Pointer to Ethernet device structure.
1223  * @param stats
1224  *   Pointer to xstats table.
1225  * @param n
1226  *   Number of entries in xstats table.
1227  * @return
1228  *   Negative value on error, number of read xstats otherwise.
1229  */
1230 static int
1231 mrvl_xstats_get(struct rte_eth_dev *dev,
1232                 struct rte_eth_xstat *stats, unsigned int n)
1233 {
1234         struct mrvl_priv *priv = dev->data->dev_private;
1235         struct pp2_ppio_statistics ppio_stats;
1236         unsigned int i;
1237
1238         if (!stats)
1239                 return 0;
1240
1241         pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1242         for (i = 0; i < n && i < RTE_DIM(mrvl_xstats_tbl); i++) {
1243                 uint64_t val;
1244
1245                 if (mrvl_xstats_tbl[i].size == sizeof(uint32_t))
1246                         val = *(uint32_t *)((uint8_t *)&ppio_stats +
1247                                             mrvl_xstats_tbl[i].offset);
1248                 else if (mrvl_xstats_tbl[i].size == sizeof(uint64_t))
1249                         val = *(uint64_t *)((uint8_t *)&ppio_stats +
1250                                             mrvl_xstats_tbl[i].offset);
1251                 else
1252                         return -EINVAL;
1253
1254                 stats[i].id = i;
1255                 stats[i].value = val;
1256         }
1257
1258         return n;
1259 }
1260
1261 /**
1262  * DPDK callback to reset extended statistics.
1263  *
1264  * @param dev
1265  *   Pointer to Ethernet device structure.
1266  */
1267 static void
1268 mrvl_xstats_reset(struct rte_eth_dev *dev)
1269 {
1270         mrvl_stats_reset(dev);
1271 }
1272
1273 /**
1274  * DPDK callback to get extended statistics names.
1275  *
1276  * @param dev (unused)
1277  *   Pointer to Ethernet device structure.
1278  * @param xstats_names
1279  *   Pointer to xstats names table.
1280  * @param size
1281  *   Size of the xstats names table.
1282  * @return
1283  *   Number of read names.
1284  */
1285 static int
1286 mrvl_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
1287                       struct rte_eth_xstat_name *xstats_names,
1288                       unsigned int size)
1289 {
1290         unsigned int i;
1291
1292         if (!xstats_names)
1293                 return RTE_DIM(mrvl_xstats_tbl);
1294
1295         for (i = 0; i < size && i < RTE_DIM(mrvl_xstats_tbl); i++)
1296                 snprintf(xstats_names[i].name, RTE_ETH_XSTATS_NAME_SIZE, "%s",
1297                          mrvl_xstats_tbl[i].name);
1298
1299         return size;
1300 }
1301
1302 /**
1303  * DPDK callback to get information about the device.
1304  *
1305  * @param dev
1306  *   Pointer to Ethernet device structure (unused).
1307  * @param info
1308  *   Info structure output buffer.
1309  */
1310 static void
1311 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
1312                    struct rte_eth_dev_info *info)
1313 {
1314         info->speed_capa = ETH_LINK_SPEED_10M |
1315                            ETH_LINK_SPEED_100M |
1316                            ETH_LINK_SPEED_1G |
1317                            ETH_LINK_SPEED_10G;
1318
1319         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1320         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1321         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1322
1323         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1324         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1325         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1326
1327         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1328         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1329         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1330
1331         info->rx_offload_capa = MRVL_RX_OFFLOADS;
1332         info->rx_queue_offload_capa = MRVL_RX_OFFLOADS;
1333
1334         info->tx_offload_capa = MRVL_TX_OFFLOADS;
1335         info->tx_queue_offload_capa = MRVL_TX_OFFLOADS;
1336
1337         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1338                                        ETH_RSS_NONFRAG_IPV4_TCP |
1339                                        ETH_RSS_NONFRAG_IPV4_UDP;
1340
1341         /* By default packets are dropped if no descriptors are available */
1342         info->default_rxconf.rx_drop_en = 1;
1343
1344         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1345 }
1346
1347 /**
1348  * Return supported packet types.
1349  *
1350  * @param dev
1351  *   Pointer to Ethernet device structure (unused).
1352  *
1353  * @return
1354  *   Const pointer to the table with supported packet types.
1355  */
1356 static const uint32_t *
1357 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1358 {
1359         static const uint32_t ptypes[] = {
1360                 RTE_PTYPE_L2_ETHER,
1361                 RTE_PTYPE_L2_ETHER_VLAN,
1362                 RTE_PTYPE_L2_ETHER_QINQ,
1363                 RTE_PTYPE_L3_IPV4,
1364                 RTE_PTYPE_L3_IPV4_EXT,
1365                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1366                 RTE_PTYPE_L3_IPV6,
1367                 RTE_PTYPE_L3_IPV6_EXT,
1368                 RTE_PTYPE_L2_ETHER_ARP,
1369                 RTE_PTYPE_L4_TCP,
1370                 RTE_PTYPE_L4_UDP
1371         };
1372
1373         return ptypes;
1374 }
1375
1376 /**
1377  * DPDK callback to get information about specific receive queue.
1378  *
1379  * @param dev
1380  *   Pointer to Ethernet device structure.
1381  * @param rx_queue_id
1382  *   Receive queue index.
1383  * @param qinfo
1384  *   Receive queue information structure.
1385  */
1386 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1387                               struct rte_eth_rxq_info *qinfo)
1388 {
1389         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1390         struct mrvl_priv *priv = dev->data->dev_private;
1391         int inq = priv->rxq_map[rx_queue_id].inq;
1392         int tc = priv->rxq_map[rx_queue_id].tc;
1393         struct pp2_ppio_tc_params *tc_params =
1394                 &priv->ppio_params.inqs_params.tcs_params[tc];
1395
1396         qinfo->mp = q->mp;
1397         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1398 }
1399
1400 /**
1401  * DPDK callback to get information about specific transmit queue.
1402  *
1403  * @param dev
1404  *   Pointer to Ethernet device structure.
1405  * @param tx_queue_id
1406  *   Transmit queue index.
1407  * @param qinfo
1408  *   Transmit queue information structure.
1409  */
1410 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1411                               struct rte_eth_txq_info *qinfo)
1412 {
1413         struct mrvl_priv *priv = dev->data->dev_private;
1414         struct mrvl_txq *txq = dev->data->tx_queues[tx_queue_id];
1415
1416         qinfo->nb_desc =
1417                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1418         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1419 }
1420
1421 /**
1422  * DPDK callback to Configure a VLAN filter.
1423  *
1424  * @param dev
1425  *   Pointer to Ethernet device structure.
1426  * @param vlan_id
1427  *   VLAN ID to filter.
1428  * @param on
1429  *   Toggle filter.
1430  *
1431  * @return
1432  *   0 on success, negative error value otherwise.
1433  */
1434 static int
1435 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1436 {
1437         struct mrvl_priv *priv = dev->data->dev_private;
1438
1439         if (!priv->ppio)
1440                 return -EPERM;
1441
1442         if (priv->isolated)
1443                 return -ENOTSUP;
1444
1445         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1446                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1447 }
1448
1449 /**
1450  * Release buffers to hardware bpool (buffer-pool)
1451  *
1452  * @param rxq
1453  *   Receive queue pointer.
1454  * @param num
1455  *   Number of buffers to release to bpool.
1456  *
1457  * @return
1458  *   0 on success, negative error value otherwise.
1459  */
1460 static int
1461 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1462 {
1463         struct buff_release_entry entries[MRVL_PP2_RXD_MAX];
1464         struct rte_mbuf *mbufs[MRVL_PP2_RXD_MAX];
1465         int i, ret;
1466         unsigned int core_id;
1467         struct pp2_hif *hif;
1468         struct pp2_bpool *bpool;
1469
1470         core_id = rte_lcore_id();
1471         if (core_id == LCORE_ID_ANY)
1472                 core_id = 0;
1473
1474         hif = mrvl_get_hif(rxq->priv, core_id);
1475         if (!hif)
1476                 return -1;
1477
1478         bpool = rxq->priv->bpool;
1479
1480         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1481         if (ret)
1482                 return ret;
1483
1484         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1485                 cookie_addr_high =
1486                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1487
1488         for (i = 0; i < num; i++) {
1489                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1490                         != cookie_addr_high) {
1491                         MRVL_LOG(ERR,
1492                                 "mbuf virtual addr high 0x%lx out of range",
1493                                 (uint64_t)mbufs[i] >> 32);
1494                         goto out;
1495                 }
1496
1497                 entries[i].buff.addr =
1498                         rte_mbuf_data_iova_default(mbufs[i]);
1499                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
1500                 entries[i].bpool = bpool;
1501         }
1502
1503         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1504         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1505
1506         if (i != num)
1507                 goto out;
1508
1509         return 0;
1510 out:
1511         for (; i < num; i++)
1512                 rte_pktmbuf_free(mbufs[i]);
1513
1514         return -1;
1515 }
1516
1517 /**
1518  * DPDK callback to configure the receive queue.
1519  *
1520  * @param dev
1521  *   Pointer to Ethernet device structure.
1522  * @param idx
1523  *   RX queue index.
1524  * @param desc
1525  *   Number of descriptors to configure in queue.
1526  * @param socket
1527  *   NUMA socket on which memory must be allocated.
1528  * @param conf
1529  *   Thresholds parameters.
1530  * @param mp
1531  *   Memory pool for buffer allocations.
1532  *
1533  * @return
1534  *   0 on success, negative error value otherwise.
1535  */
1536 static int
1537 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1538                     unsigned int socket,
1539                     const struct rte_eth_rxconf *conf,
1540                     struct rte_mempool *mp)
1541 {
1542         struct mrvl_priv *priv = dev->data->dev_private;
1543         struct mrvl_rxq *rxq;
1544         uint32_t min_size,
1545                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1546         int ret, tc, inq;
1547         uint64_t offloads;
1548
1549         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1550
1551         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1552                 /*
1553                  * Unknown TC mapping, mapping will not have a correct queue.
1554                  */
1555                 MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu",
1556                         idx, priv->ppio_id);
1557                 return -EFAULT;
1558         }
1559
1560         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
1561                    MRVL_PKT_EFFEC_OFFS;
1562         if (min_size < max_rx_pkt_len) {
1563                 MRVL_LOG(ERR,
1564                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.",
1565                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
1566                         MRVL_PKT_EFFEC_OFFS,
1567                         max_rx_pkt_len);
1568                 return -EINVAL;
1569         }
1570
1571         if (dev->data->rx_queues[idx]) {
1572                 rte_free(dev->data->rx_queues[idx]);
1573                 dev->data->rx_queues[idx] = NULL;
1574         }
1575
1576         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1577         if (!rxq)
1578                 return -ENOMEM;
1579
1580         rxq->priv = priv;
1581         rxq->mp = mp;
1582         rxq->cksum_enabled = offloads & DEV_RX_OFFLOAD_IPV4_CKSUM;
1583         rxq->queue_id = idx;
1584         rxq->port_id = dev->data->port_id;
1585         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1586
1587         tc = priv->rxq_map[rxq->queue_id].tc,
1588         inq = priv->rxq_map[rxq->queue_id].inq;
1589         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1590                 desc;
1591
1592         ret = mrvl_fill_bpool(rxq, desc);
1593         if (ret) {
1594                 rte_free(rxq);
1595                 return ret;
1596         }
1597
1598         priv->bpool_init_size += desc;
1599
1600         dev->data->rx_queues[idx] = rxq;
1601
1602         return 0;
1603 }
1604
1605 /**
1606  * DPDK callback to release the receive queue.
1607  *
1608  * @param rxq
1609  *   Generic receive queue pointer.
1610  */
1611 static void
1612 mrvl_rx_queue_release(void *rxq)
1613 {
1614         struct mrvl_rxq *q = rxq;
1615         struct pp2_ppio_tc_params *tc_params;
1616         int i, num, tc, inq;
1617         struct pp2_hif *hif;
1618         unsigned int core_id = rte_lcore_id();
1619
1620         if (core_id == LCORE_ID_ANY)
1621                 core_id = 0;
1622
1623         if (!q)
1624                 return;
1625
1626         hif = mrvl_get_hif(q->priv, core_id);
1627
1628         if (!hif)
1629                 return;
1630
1631         tc = q->priv->rxq_map[q->queue_id].tc;
1632         inq = q->priv->rxq_map[q->queue_id].inq;
1633         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1634         num = tc_params->inqs_params[inq].size;
1635         for (i = 0; i < num; i++) {
1636                 struct pp2_buff_inf inf;
1637                 uint64_t addr;
1638
1639                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
1640                 addr = cookie_addr_high | inf.cookie;
1641                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1642         }
1643
1644         rte_free(q);
1645 }
1646
1647 /**
1648  * DPDK callback to configure the transmit queue.
1649  *
1650  * @param dev
1651  *   Pointer to Ethernet device structure.
1652  * @param idx
1653  *   Transmit queue index.
1654  * @param desc
1655  *   Number of descriptors to configure in the queue.
1656  * @param socket
1657  *   NUMA socket on which memory must be allocated.
1658  * @param conf
1659  *   Tx queue configuration parameters.
1660  *
1661  * @return
1662  *   0 on success, negative error value otherwise.
1663  */
1664 static int
1665 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1666                     unsigned int socket,
1667                     const struct rte_eth_txconf *conf)
1668 {
1669         struct mrvl_priv *priv = dev->data->dev_private;
1670         struct mrvl_txq *txq;
1671
1672         if (dev->data->tx_queues[idx]) {
1673                 rte_free(dev->data->tx_queues[idx]);
1674                 dev->data->tx_queues[idx] = NULL;
1675         }
1676
1677         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1678         if (!txq)
1679                 return -ENOMEM;
1680
1681         txq->priv = priv;
1682         txq->queue_id = idx;
1683         txq->port_id = dev->data->port_id;
1684         txq->tx_deferred_start = conf->tx_deferred_start;
1685         dev->data->tx_queues[idx] = txq;
1686
1687         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1688
1689         return 0;
1690 }
1691
1692 /**
1693  * DPDK callback to release the transmit queue.
1694  *
1695  * @param txq
1696  *   Generic transmit queue pointer.
1697  */
1698 static void
1699 mrvl_tx_queue_release(void *txq)
1700 {
1701         struct mrvl_txq *q = txq;
1702
1703         if (!q)
1704                 return;
1705
1706         rte_free(q);
1707 }
1708
1709 /**
1710  * DPDK callback to get flow control configuration.
1711  *
1712  * @param dev
1713  *  Pointer to Ethernet device structure.
1714  * @param fc_conf
1715  *  Pointer to the flow control configuration.
1716  *
1717  * @return
1718  *  0 on success, negative error value otherwise.
1719  */
1720 static int
1721 mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1722 {
1723         struct mrvl_priv *priv = dev->data->dev_private;
1724         int ret, en;
1725
1726         if (!priv)
1727                 return -EPERM;
1728
1729         ret = pp2_ppio_get_rx_pause(priv->ppio, &en);
1730         if (ret) {
1731                 MRVL_LOG(ERR, "Failed to read rx pause state");
1732                 return ret;
1733         }
1734
1735         fc_conf->mode = en ? RTE_FC_RX_PAUSE : RTE_FC_NONE;
1736
1737         return 0;
1738 }
1739
1740 /**
1741  * DPDK callback to set flow control configuration.
1742  *
1743  * @param dev
1744  *  Pointer to Ethernet device structure.
1745  * @param fc_conf
1746  *  Pointer to the flow control configuration.
1747  *
1748  * @return
1749  *  0 on success, negative error value otherwise.
1750  */
1751 static int
1752 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1753 {
1754         struct mrvl_priv *priv = dev->data->dev_private;
1755
1756         if (!priv)
1757                 return -EPERM;
1758
1759         if (fc_conf->high_water ||
1760             fc_conf->low_water ||
1761             fc_conf->pause_time ||
1762             fc_conf->mac_ctrl_frame_fwd ||
1763             fc_conf->autoneg) {
1764                 MRVL_LOG(ERR, "Flowctrl parameter is not supported");
1765
1766                 return -EINVAL;
1767         }
1768
1769         if (fc_conf->mode == RTE_FC_NONE ||
1770             fc_conf->mode == RTE_FC_RX_PAUSE) {
1771                 int ret, en;
1772
1773                 en = fc_conf->mode == RTE_FC_NONE ? 0 : 1;
1774                 ret = pp2_ppio_set_rx_pause(priv->ppio, en);
1775                 if (ret)
1776                         MRVL_LOG(ERR,
1777                                 "Failed to change flowctrl on RX side");
1778
1779                 return ret;
1780         }
1781
1782         return 0;
1783 }
1784
1785 /**
1786  * Update RSS hash configuration
1787  *
1788  * @param dev
1789  *   Pointer to Ethernet device structure.
1790  * @param rss_conf
1791  *   Pointer to RSS configuration.
1792  *
1793  * @return
1794  *   0 on success, negative error value otherwise.
1795  */
1796 static int
1797 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1798                      struct rte_eth_rss_conf *rss_conf)
1799 {
1800         struct mrvl_priv *priv = dev->data->dev_private;
1801
1802         if (priv->isolated)
1803                 return -ENOTSUP;
1804
1805         return mrvl_configure_rss(priv, rss_conf);
1806 }
1807
1808 /**
1809  * DPDK callback to get RSS hash configuration.
1810  *
1811  * @param dev
1812  *   Pointer to Ethernet device structure.
1813  * @rss_conf
1814  *   Pointer to RSS configuration.
1815  *
1816  * @return
1817  *   Always 0.
1818  */
1819 static int
1820 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1821                        struct rte_eth_rss_conf *rss_conf)
1822 {
1823         struct mrvl_priv *priv = dev->data->dev_private;
1824         enum pp2_ppio_hash_type hash_type =
1825                 priv->ppio_params.inqs_params.hash_type;
1826
1827         rss_conf->rss_key = NULL;
1828
1829         if (hash_type == PP2_PPIO_HASH_T_NONE)
1830                 rss_conf->rss_hf = 0;
1831         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1832                 rss_conf->rss_hf = ETH_RSS_IPV4;
1833         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1834                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1835         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1836                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1837
1838         return 0;
1839 }
1840
1841 /**
1842  * DPDK callback to get rte_flow callbacks.
1843  *
1844  * @param dev
1845  *   Pointer to the device structure.
1846  * @param filer_type
1847  *   Flow filter type.
1848  * @param filter_op
1849  *   Flow filter operation.
1850  * @param arg
1851  *   Pointer to pass the flow ops.
1852  *
1853  * @return
1854  *   0 on success, negative error value otherwise.
1855  */
1856 static int
1857 mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
1858                      enum rte_filter_type filter_type,
1859                      enum rte_filter_op filter_op, void *arg)
1860 {
1861         switch (filter_type) {
1862         case RTE_ETH_FILTER_GENERIC:
1863                 if (filter_op != RTE_ETH_FILTER_GET)
1864                         return -EINVAL;
1865                 *(const void **)arg = &mrvl_flow_ops;
1866                 return 0;
1867         default:
1868                 MRVL_LOG(WARNING, "Filter type (%d) not supported",
1869                                 filter_type);
1870                 return -EINVAL;
1871         }
1872 }
1873
1874 /**
1875  * DPDK callback to get rte_mtr callbacks.
1876  *
1877  * @param dev
1878  *   Pointer to the device structure.
1879  * @param ops
1880  *   Pointer to pass the mtr ops.
1881  *
1882  * @return
1883  *   Always 0.
1884  */
1885 static int
1886 mrvl_mtr_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
1887 {
1888         *(const void **)ops = &mrvl_mtr_ops;
1889
1890         return 0;
1891 }
1892
1893 /**
1894  * DPDK callback to get rte_tm callbacks.
1895  *
1896  * @param dev
1897  *   Pointer to the device structure.
1898  * @param ops
1899  *   Pointer to pass the tm ops.
1900  *
1901  * @return
1902  *   Always 0.
1903  */
1904 static int
1905 mrvl_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
1906 {
1907         *(const void **)ops = &mrvl_tm_ops;
1908
1909         return 0;
1910 }
1911
1912 static const struct eth_dev_ops mrvl_ops = {
1913         .dev_configure = mrvl_dev_configure,
1914         .dev_start = mrvl_dev_start,
1915         .dev_stop = mrvl_dev_stop,
1916         .dev_set_link_up = mrvl_dev_set_link_up,
1917         .dev_set_link_down = mrvl_dev_set_link_down,
1918         .dev_close = mrvl_dev_close,
1919         .link_update = mrvl_link_update,
1920         .promiscuous_enable = mrvl_promiscuous_enable,
1921         .allmulticast_enable = mrvl_allmulticast_enable,
1922         .promiscuous_disable = mrvl_promiscuous_disable,
1923         .allmulticast_disable = mrvl_allmulticast_disable,
1924         .mac_addr_remove = mrvl_mac_addr_remove,
1925         .mac_addr_add = mrvl_mac_addr_add,
1926         .mac_addr_set = mrvl_mac_addr_set,
1927         .mtu_set = mrvl_mtu_set,
1928         .stats_get = mrvl_stats_get,
1929         .stats_reset = mrvl_stats_reset,
1930         .xstats_get = mrvl_xstats_get,
1931         .xstats_reset = mrvl_xstats_reset,
1932         .xstats_get_names = mrvl_xstats_get_names,
1933         .dev_infos_get = mrvl_dev_infos_get,
1934         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
1935         .rxq_info_get = mrvl_rxq_info_get,
1936         .txq_info_get = mrvl_txq_info_get,
1937         .vlan_filter_set = mrvl_vlan_filter_set,
1938         .tx_queue_start = mrvl_tx_queue_start,
1939         .tx_queue_stop = mrvl_tx_queue_stop,
1940         .rx_queue_setup = mrvl_rx_queue_setup,
1941         .rx_queue_release = mrvl_rx_queue_release,
1942         .tx_queue_setup = mrvl_tx_queue_setup,
1943         .tx_queue_release = mrvl_tx_queue_release,
1944         .flow_ctrl_get = mrvl_flow_ctrl_get,
1945         .flow_ctrl_set = mrvl_flow_ctrl_set,
1946         .rss_hash_update = mrvl_rss_hash_update,
1947         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1948         .filter_ctrl = mrvl_eth_filter_ctrl,
1949         .mtr_ops_get = mrvl_mtr_ops_get,
1950         .tm_ops_get = mrvl_tm_ops_get,
1951 };
1952
1953 /**
1954  * Return packet type information and l3/l4 offsets.
1955  *
1956  * @param desc
1957  *   Pointer to the received packet descriptor.
1958  * @param l3_offset
1959  *   l3 packet offset.
1960  * @param l4_offset
1961  *   l4 packet offset.
1962  *
1963  * @return
1964  *   Packet type information.
1965  */
1966 static inline uint64_t
1967 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
1968                                     uint8_t *l3_offset, uint8_t *l4_offset)
1969 {
1970         enum pp2_inq_l3_type l3_type;
1971         enum pp2_inq_l4_type l4_type;
1972         enum pp2_inq_vlan_tag vlan_tag;
1973         uint64_t packet_type;
1974
1975         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
1976         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
1977         pp2_ppio_inq_desc_get_vlan_tag(desc, &vlan_tag);
1978
1979         packet_type = RTE_PTYPE_L2_ETHER;
1980
1981         switch (vlan_tag) {
1982         case PP2_INQ_VLAN_TAG_SINGLE:
1983                 packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
1984                 break;
1985         case PP2_INQ_VLAN_TAG_DOUBLE:
1986         case PP2_INQ_VLAN_TAG_TRIPLE:
1987                 packet_type |= RTE_PTYPE_L2_ETHER_QINQ;
1988                 break;
1989         default:
1990                 break;
1991         }
1992
1993         switch (l3_type) {
1994         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
1995                 packet_type |= RTE_PTYPE_L3_IPV4;
1996                 break;
1997         case PP2_INQ_L3_TYPE_IPV4_OK:
1998                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
1999                 break;
2000         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
2001                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
2002                 break;
2003         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
2004                 packet_type |= RTE_PTYPE_L3_IPV6;
2005                 break;
2006         case PP2_INQ_L3_TYPE_IPV6_EXT:
2007                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
2008                 break;
2009         case PP2_INQ_L3_TYPE_ARP:
2010                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
2011                 /*
2012                  * In case of ARP l4_offset is set to wrong value.
2013                  * Set it to proper one so that later on mbuf->l3_len can be
2014                  * calculated subtracting l4_offset and l3_offset.
2015                  */
2016                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
2017                 break;
2018         default:
2019                 MRVL_LOG(DEBUG, "Failed to recognise l3 packet type");
2020                 break;
2021         }
2022
2023         switch (l4_type) {
2024         case PP2_INQ_L4_TYPE_TCP:
2025                 packet_type |= RTE_PTYPE_L4_TCP;
2026                 break;
2027         case PP2_INQ_L4_TYPE_UDP:
2028                 packet_type |= RTE_PTYPE_L4_UDP;
2029                 break;
2030         default:
2031                 MRVL_LOG(DEBUG, "Failed to recognise l4 packet type");
2032                 break;
2033         }
2034
2035         return packet_type;
2036 }
2037
2038 /**
2039  * Get offload information from the received packet descriptor.
2040  *
2041  * @param desc
2042  *   Pointer to the received packet descriptor.
2043  *
2044  * @return
2045  *   Mbuf offload flags.
2046  */
2047 static inline uint64_t
2048 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
2049 {
2050         uint64_t flags;
2051         enum pp2_inq_desc_status status;
2052
2053         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
2054         if (unlikely(status != PP2_DESC_ERR_OK))
2055                 flags = PKT_RX_IP_CKSUM_BAD;
2056         else
2057                 flags = PKT_RX_IP_CKSUM_GOOD;
2058
2059         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
2060         if (unlikely(status != PP2_DESC_ERR_OK))
2061                 flags |= PKT_RX_L4_CKSUM_BAD;
2062         else
2063                 flags |= PKT_RX_L4_CKSUM_GOOD;
2064
2065         return flags;
2066 }
2067
2068 /**
2069  * DPDK callback for receive.
2070  *
2071  * @param rxq
2072  *   Generic pointer to the receive queue.
2073  * @param rx_pkts
2074  *   Array to store received packets.
2075  * @param nb_pkts
2076  *   Maximum number of packets in array.
2077  *
2078  * @return
2079  *   Number of packets successfully received.
2080  */
2081 static uint16_t
2082 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2083 {
2084         struct mrvl_rxq *q = rxq;
2085         struct pp2_ppio_desc descs[nb_pkts];
2086         struct pp2_bpool *bpool;
2087         int i, ret, rx_done = 0;
2088         int num;
2089         struct pp2_hif *hif;
2090         unsigned int core_id = rte_lcore_id();
2091
2092         hif = mrvl_get_hif(q->priv, core_id);
2093
2094         if (unlikely(!q->priv->ppio || !hif))
2095                 return 0;
2096
2097         bpool = q->priv->bpool;
2098
2099         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
2100                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
2101         if (unlikely(ret < 0)) {
2102                 MRVL_LOG(ERR, "Failed to receive packets");
2103                 return 0;
2104         }
2105         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
2106
2107         for (i = 0; i < nb_pkts; i++) {
2108                 struct rte_mbuf *mbuf;
2109                 uint8_t l3_offset, l4_offset;
2110                 enum pp2_inq_desc_status status;
2111                 uint64_t addr;
2112
2113                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2114                         struct pp2_ppio_desc *pref_desc;
2115                         u64 pref_addr;
2116
2117                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
2118                         pref_addr = cookie_addr_high |
2119                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
2120                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
2121                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
2122                 }
2123
2124                 addr = cookie_addr_high |
2125                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
2126                 mbuf = (struct rte_mbuf *)addr;
2127                 rte_pktmbuf_reset(mbuf);
2128
2129                 /* drop packet in case of mac, overrun or resource error */
2130                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
2131                 if (unlikely(status != PP2_DESC_ERR_OK)) {
2132                         struct pp2_buff_inf binf = {
2133                                 .addr = rte_mbuf_data_iova_default(mbuf),
2134                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
2135                         };
2136
2137                         pp2_bpool_put_buff(hif, bpool, &binf);
2138                         mrvl_port_bpool_size
2139                                 [bpool->pp2_id][bpool->id][core_id]++;
2140                         q->drop_mac++;
2141                         continue;
2142                 }
2143
2144                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
2145                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
2146                 mbuf->data_len = mbuf->pkt_len;
2147                 mbuf->port = q->port_id;
2148                 mbuf->packet_type =
2149                         mrvl_desc_to_packet_type_and_offset(&descs[i],
2150                                                             &l3_offset,
2151                                                             &l4_offset);
2152                 mbuf->l2_len = l3_offset;
2153                 mbuf->l3_len = l4_offset - l3_offset;
2154
2155                 if (likely(q->cksum_enabled))
2156                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
2157
2158                 rx_pkts[rx_done++] = mbuf;
2159                 q->bytes_recv += mbuf->pkt_len;
2160         }
2161
2162         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
2163                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
2164
2165                 if (unlikely(num <= q->priv->bpool_min_size ||
2166                              (!rx_done && num < q->priv->bpool_init_size))) {
2167                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
2168                         if (ret)
2169                                 MRVL_LOG(ERR, "Failed to fill bpool");
2170                 } else if (unlikely(num > q->priv->bpool_max_size)) {
2171                         int i;
2172                         int pkt_to_remove = num - q->priv->bpool_init_size;
2173                         struct rte_mbuf *mbuf;
2174                         struct pp2_buff_inf buff;
2175
2176                         MRVL_LOG(DEBUG,
2177                                 "port-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)",
2178                                 bpool->pp2_id, q->priv->ppio->port_id,
2179                                 bpool->id, pkt_to_remove, num,
2180                                 q->priv->bpool_init_size);
2181
2182                         for (i = 0; i < pkt_to_remove; i++) {
2183                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
2184                                 if (ret)
2185                                         break;
2186                                 mbuf = (struct rte_mbuf *)
2187                                         (cookie_addr_high | buff.cookie);
2188                                 rte_pktmbuf_free(mbuf);
2189                         }
2190                         mrvl_port_bpool_size
2191                                 [bpool->pp2_id][bpool->id][core_id] -= i;
2192                 }
2193                 rte_spinlock_unlock(&q->priv->lock);
2194         }
2195
2196         return rx_done;
2197 }
2198
2199 /**
2200  * Prepare offload information.
2201  *
2202  * @param ol_flags
2203  *   Offload flags.
2204  * @param packet_type
2205  *   Packet type bitfield.
2206  * @param l3_type
2207  *   Pointer to the pp2_ouq_l3_type structure.
2208  * @param l4_type
2209  *   Pointer to the pp2_outq_l4_type structure.
2210  * @param gen_l3_cksum
2211  *   Will be set to 1 in case l3 checksum is computed.
2212  * @param l4_cksum
2213  *   Will be set to 1 in case l4 checksum is computed.
2214  *
2215  * @return
2216  *   0 on success, negative error value otherwise.
2217  */
2218 static inline int
2219 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
2220                         enum pp2_outq_l3_type *l3_type,
2221                         enum pp2_outq_l4_type *l4_type,
2222                         int *gen_l3_cksum,
2223                         int *gen_l4_cksum)
2224 {
2225         /*
2226          * Based on ol_flags prepare information
2227          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
2228          * for offloading.
2229          */
2230         if (ol_flags & PKT_TX_IPV4) {
2231                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
2232                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
2233         } else if (ol_flags & PKT_TX_IPV6) {
2234                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
2235                 /* no checksum for ipv6 header */
2236                 *gen_l3_cksum = 0;
2237         } else {
2238                 /* if something different then stop processing */
2239                 return -1;
2240         }
2241
2242         ol_flags &= PKT_TX_L4_MASK;
2243         if ((packet_type & RTE_PTYPE_L4_TCP) &&
2244             ol_flags == PKT_TX_TCP_CKSUM) {
2245                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
2246                 *gen_l4_cksum = 1;
2247         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
2248                    ol_flags == PKT_TX_UDP_CKSUM) {
2249                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
2250                 *gen_l4_cksum = 1;
2251         } else {
2252                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
2253                 /* no checksum for other type */
2254                 *gen_l4_cksum = 0;
2255         }
2256
2257         return 0;
2258 }
2259
2260 /**
2261  * Release already sent buffers to bpool (buffer-pool).
2262  *
2263  * @param ppio
2264  *   Pointer to the port structure.
2265  * @param hif
2266  *   Pointer to the MUSDK hardware interface.
2267  * @param sq
2268  *   Pointer to the shadow queue.
2269  * @param qid
2270  *   Queue id number.
2271  * @param force
2272  *   Force releasing packets.
2273  */
2274 static inline void
2275 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
2276                        unsigned int core_id, struct mrvl_shadow_txq *sq,
2277                        int qid, int force)
2278 {
2279         struct buff_release_entry *entry;
2280         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
2281         int i;
2282
2283         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
2284
2285         sq->num_to_release += nb_done;
2286
2287         if (likely(!force &&
2288                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
2289                 return;
2290
2291         nb_done = sq->num_to_release;
2292         sq->num_to_release = 0;
2293
2294         for (i = 0; i < nb_done; i++) {
2295                 entry = &sq->ent[sq->tail + num];
2296                 if (unlikely(!entry->buff.addr)) {
2297                         MRVL_LOG(ERR,
2298                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!",
2299                                 sq->tail, (u64)entry->buff.cookie,
2300                                 (u64)entry->buff.addr);
2301                         skip_bufs = 1;
2302                         goto skip;
2303                 }
2304
2305                 if (unlikely(!entry->bpool)) {
2306                         struct rte_mbuf *mbuf;
2307
2308                         mbuf = (struct rte_mbuf *)
2309                                (cookie_addr_high | entry->buff.cookie);
2310                         rte_pktmbuf_free(mbuf);
2311                         skip_bufs = 1;
2312                         goto skip;
2313                 }
2314
2315                 mrvl_port_bpool_size
2316                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
2317                 num++;
2318                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
2319                         goto skip;
2320                 continue;
2321 skip:
2322                 if (likely(num))
2323                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2324                 num += skip_bufs;
2325                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2326                 sq->size -= num;
2327                 num = 0;
2328                 skip_bufs = 0;
2329         }
2330
2331         if (likely(num)) {
2332                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2333                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2334                 sq->size -= num;
2335         }
2336 }
2337
2338 /**
2339  * DPDK callback for transmit.
2340  *
2341  * @param txq
2342  *   Generic pointer transmit queue.
2343  * @param tx_pkts
2344  *   Packets to transmit.
2345  * @param nb_pkts
2346  *   Number of packets in array.
2347  *
2348  * @return
2349  *   Number of packets successfully transmitted.
2350  */
2351 static uint16_t
2352 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2353 {
2354         struct mrvl_txq *q = txq;
2355         struct mrvl_shadow_txq *sq;
2356         struct pp2_hif *hif;
2357         struct pp2_ppio_desc descs[nb_pkts];
2358         unsigned int core_id = rte_lcore_id();
2359         int i, ret, bytes_sent = 0;
2360         uint16_t num, sq_free_size;
2361         uint64_t addr;
2362
2363         hif = mrvl_get_hif(q->priv, core_id);
2364         sq = &q->shadow_txqs[core_id];
2365
2366         if (unlikely(!q->priv->ppio || !hif))
2367                 return 0;
2368
2369         if (sq->size)
2370                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2371                                        sq, q->queue_id, 0);
2372
2373         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2374         if (unlikely(nb_pkts > sq_free_size)) {
2375                 MRVL_LOG(DEBUG,
2376                         "No room in shadow queue for %d packets! %d packets will be sent.",
2377                         nb_pkts, sq_free_size);
2378                 nb_pkts = sq_free_size;
2379         }
2380
2381         for (i = 0; i < nb_pkts; i++) {
2382                 struct rte_mbuf *mbuf = tx_pkts[i];
2383                 int gen_l3_cksum, gen_l4_cksum;
2384                 enum pp2_outq_l3_type l3_type;
2385                 enum pp2_outq_l4_type l4_type;
2386
2387                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2388                         struct rte_mbuf *pref_pkt_hdr;
2389
2390                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2391                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2392                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2393                 }
2394
2395                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
2396                 sq->ent[sq->head].buff.addr =
2397                         rte_mbuf_data_iova_default(mbuf);
2398                 sq->ent[sq->head].bpool =
2399                         (unlikely(mbuf->port >= RTE_MAX_ETHPORTS ||
2400                          mbuf->refcnt > 1)) ? NULL :
2401                          mrvl_port_to_bpool_lookup[mbuf->port];
2402                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
2403                 sq->size++;
2404
2405                 pp2_ppio_outq_desc_reset(&descs[i]);
2406                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
2407                                                  rte_pktmbuf_iova(mbuf));
2408                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
2409                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
2410                                                rte_pktmbuf_pkt_len(mbuf));
2411
2412                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2413                 /*
2414                  * in case unsupported ol_flags were passed
2415                  * do not update descriptor offload information
2416                  */
2417                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
2418                                               &l3_type, &l4_type, &gen_l3_cksum,
2419                                               &gen_l4_cksum);
2420                 if (unlikely(ret))
2421                         continue;
2422
2423                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2424                                                   mbuf->l2_len,
2425                                                   mbuf->l2_len + mbuf->l3_len,
2426                                                   gen_l3_cksum, gen_l4_cksum);
2427         }
2428
2429         num = nb_pkts;
2430         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2431         /* number of packets that were not sent */
2432         if (unlikely(num > nb_pkts)) {
2433                 for (i = nb_pkts; i < num; i++) {
2434                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2435                                 MRVL_PP2_TX_SHADOWQ_MASK;
2436                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
2437                         bytes_sent -=
2438                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2439                 }
2440                 sq->size -= num - nb_pkts;
2441         }
2442
2443         q->bytes_sent += bytes_sent;
2444
2445         return nb_pkts;
2446 }
2447
2448 /**
2449  * Initialize packet processor.
2450  *
2451  * @return
2452  *   0 on success, negative error value otherwise.
2453  */
2454 static int
2455 mrvl_init_pp2(void)
2456 {
2457         struct pp2_init_params init_params;
2458
2459         memset(&init_params, 0, sizeof(init_params));
2460         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
2461         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
2462         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
2463
2464         return pp2_init(&init_params);
2465 }
2466
2467 /**
2468  * Deinitialize packet processor.
2469  *
2470  * @return
2471  *   0 on success, negative error value otherwise.
2472  */
2473 static void
2474 mrvl_deinit_pp2(void)
2475 {
2476         pp2_deinit();
2477 }
2478
2479 /**
2480  * Create private device structure.
2481  *
2482  * @param dev_name
2483  *   Pointer to the port name passed in the initialization parameters.
2484  *
2485  * @return
2486  *   Pointer to the newly allocated private device structure.
2487  */
2488 static struct mrvl_priv *
2489 mrvl_priv_create(const char *dev_name)
2490 {
2491         struct pp2_bpool_params bpool_params;
2492         char match[MRVL_MATCH_LEN];
2493         struct mrvl_priv *priv;
2494         int ret, bpool_bit;
2495
2496         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
2497         if (!priv)
2498                 return NULL;
2499
2500         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
2501                                        &priv->pp_id, &priv->ppio_id);
2502         if (ret)
2503                 goto out_free_priv;
2504
2505         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
2506                                      PP2_BPOOL_NUM_POOLS);
2507         if (bpool_bit < 0)
2508                 goto out_free_priv;
2509         priv->bpool_bit = bpool_bit;
2510
2511         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
2512                  priv->bpool_bit);
2513         memset(&bpool_params, 0, sizeof(bpool_params));
2514         bpool_params.match = match;
2515         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
2516         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
2517         if (ret)
2518                 goto out_clear_bpool_bit;
2519
2520         priv->ppio_params.type = PP2_PPIO_T_NIC;
2521         rte_spinlock_init(&priv->lock);
2522
2523         return priv;
2524 out_clear_bpool_bit:
2525         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2526 out_free_priv:
2527         rte_free(priv);
2528         return NULL;
2529 }
2530
2531 /**
2532  * Create device representing Ethernet port.
2533  *
2534  * @param name
2535  *   Pointer to the port's name.
2536  *
2537  * @return
2538  *   0 on success, negative error value otherwise.
2539  */
2540 static int
2541 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
2542 {
2543         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
2544         struct rte_eth_dev *eth_dev;
2545         struct mrvl_priv *priv;
2546         struct ifreq req;
2547
2548         eth_dev = rte_eth_dev_allocate(name);
2549         if (!eth_dev)
2550                 return -ENOMEM;
2551
2552         priv = mrvl_priv_create(name);
2553         if (!priv) {
2554                 ret = -ENOMEM;
2555                 goto out_free_dev;
2556         }
2557
2558         eth_dev->data->mac_addrs =
2559                 rte_zmalloc("mac_addrs",
2560                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2561         if (!eth_dev->data->mac_addrs) {
2562                 MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
2563                 ret = -ENOMEM;
2564                 goto out_free_priv;
2565         }
2566
2567         memset(&req, 0, sizeof(req));
2568         strcpy(req.ifr_name, name);
2569         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2570         if (ret)
2571                 goto out_free_mac;
2572
2573         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2574                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2575
2576         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2577         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
2578         eth_dev->data->kdrv = RTE_KDRV_NONE;
2579         eth_dev->data->dev_private = priv;
2580         eth_dev->device = &vdev->device;
2581         eth_dev->dev_ops = &mrvl_ops;
2582
2583         rte_eth_dev_probing_finish(eth_dev);
2584         return 0;
2585 out_free_mac:
2586         rte_free(eth_dev->data->mac_addrs);
2587 out_free_dev:
2588         rte_eth_dev_release_port(eth_dev);
2589 out_free_priv:
2590         rte_free(priv);
2591
2592         return ret;
2593 }
2594
2595 /**
2596  * Cleanup previously created device representing Ethernet port.
2597  *
2598  * @param name
2599  *   Pointer to the port name.
2600  */
2601 static void
2602 mrvl_eth_dev_destroy(const char *name)
2603 {
2604         struct rte_eth_dev *eth_dev;
2605         struct mrvl_priv *priv;
2606
2607         eth_dev = rte_eth_dev_allocated(name);
2608         if (!eth_dev)
2609                 return;
2610
2611         priv = eth_dev->data->dev_private;
2612         pp2_bpool_deinit(priv->bpool);
2613         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2614         rte_free(priv);
2615         rte_free(eth_dev->data->mac_addrs);
2616         rte_eth_dev_release_port(eth_dev);
2617 }
2618
2619 /**
2620  * Callback used by rte_kvargs_process() during argument parsing.
2621  *
2622  * @param key
2623  *   Pointer to the parsed key (unused).
2624  * @param value
2625  *   Pointer to the parsed value.
2626  * @param extra_args
2627  *   Pointer to the extra arguments which contains address of the
2628  *   table of pointers to parsed interface names.
2629  *
2630  * @return
2631  *   Always 0.
2632  */
2633 static int
2634 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2635                  void *extra_args)
2636 {
2637         struct mrvl_ifnames *ifnames = extra_args;
2638
2639         ifnames->names[ifnames->idx++] = value;
2640
2641         return 0;
2642 }
2643
2644 /**
2645  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2646  */
2647 static void
2648 mrvl_deinit_hifs(void)
2649 {
2650         int i;
2651
2652         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
2653                 if (hifs[i])
2654                         pp2_hif_deinit(hifs[i]);
2655         }
2656         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
2657         memset(hifs, 0, sizeof(hifs));
2658 }
2659
2660 /**
2661  * DPDK callback to register the virtual device.
2662  *
2663  * @param vdev
2664  *   Pointer to the virtual device.
2665  *
2666  * @return
2667  *   0 on success, negative error value otherwise.
2668  */
2669 static int
2670 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2671 {
2672         struct rte_kvargs *kvlist;
2673         struct mrvl_ifnames ifnames;
2674         int ret = -EINVAL;
2675         uint32_t i, ifnum, cfgnum;
2676         const char *params;
2677
2678         params = rte_vdev_device_args(vdev);
2679         if (!params)
2680                 return -EINVAL;
2681
2682         kvlist = rte_kvargs_parse(params, valid_args);
2683         if (!kvlist)
2684                 return -EINVAL;
2685
2686         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2687         if (ifnum > RTE_DIM(ifnames.names))
2688                 goto out_free_kvlist;
2689
2690         ifnames.idx = 0;
2691         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2692                            mrvl_get_ifnames, &ifnames);
2693
2694
2695         /*
2696          * The below system initialization should be done only once,
2697          * on the first provided configuration file
2698          */
2699         if (!mrvl_qos_cfg) {
2700                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2701                 MRVL_LOG(INFO, "Parsing config file!");
2702                 if (cfgnum > 1) {
2703                         MRVL_LOG(ERR, "Cannot handle more than one config file!");
2704                         goto out_free_kvlist;
2705                 } else if (cfgnum == 1) {
2706                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2707                                            mrvl_get_qoscfg, &mrvl_qos_cfg);
2708                 }
2709         }
2710
2711         if (mrvl_dev_num)
2712                 goto init_devices;
2713
2714         MRVL_LOG(INFO, "Perform MUSDK initializations");
2715
2716         ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist);
2717         if (ret)
2718                 goto out_free_kvlist;
2719
2720         ret = mrvl_init_pp2();
2721         if (ret) {
2722                 MRVL_LOG(ERR, "Failed to init PP!");
2723                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2724                 goto out_free_kvlist;
2725         }
2726
2727         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2728         memset(mrvl_port_to_bpool_lookup, 0, sizeof(mrvl_port_to_bpool_lookup));
2729
2730         mrvl_lcore_first = RTE_MAX_LCORE;
2731         mrvl_lcore_last = 0;
2732
2733 init_devices:
2734         for (i = 0; i < ifnum; i++) {
2735                 MRVL_LOG(INFO, "Creating %s", ifnames.names[i]);
2736                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
2737                 if (ret)
2738                         goto out_cleanup;
2739         }
2740         mrvl_dev_num += ifnum;
2741
2742         rte_kvargs_free(kvlist);
2743
2744         return 0;
2745 out_cleanup:
2746         for (; i > 0; i--)
2747                 mrvl_eth_dev_destroy(ifnames.names[i]);
2748
2749         if (mrvl_dev_num == 0) {
2750                 mrvl_deinit_pp2();
2751                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2752         }
2753 out_free_kvlist:
2754         rte_kvargs_free(kvlist);
2755
2756         return ret;
2757 }
2758
2759 /**
2760  * DPDK callback to remove virtual device.
2761  *
2762  * @param vdev
2763  *   Pointer to the removed virtual device.
2764  *
2765  * @return
2766  *   0 on success, negative error value otherwise.
2767  */
2768 static int
2769 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2770 {
2771         int i;
2772         const char *name;
2773
2774         name = rte_vdev_device_name(vdev);
2775         if (!name)
2776                 return -EINVAL;
2777
2778         MRVL_LOG(INFO, "Removing %s", name);
2779
2780         RTE_ETH_FOREACH_DEV(i) { /* FIXME: removing all devices! */
2781                 char ifname[RTE_ETH_NAME_MAX_LEN];
2782
2783                 rte_eth_dev_get_name_by_port(i, ifname);
2784                 mrvl_eth_dev_destroy(ifname);
2785                 mrvl_dev_num--;
2786         }
2787
2788         if (mrvl_dev_num == 0) {
2789                 MRVL_LOG(INFO, "Perform MUSDK deinit");
2790                 mrvl_deinit_hifs();
2791                 mrvl_deinit_pp2();
2792                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2793         }
2794
2795         return 0;
2796 }
2797
2798 static struct rte_vdev_driver pmd_mrvl_drv = {
2799         .probe = rte_pmd_mrvl_probe,
2800         .remove = rte_pmd_mrvl_remove,
2801 };
2802
2803 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv);
2804 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2);
2805
2806 RTE_INIT(mrvl_init_log)
2807 {
2808         mrvl_logtype = rte_log_register("pmd.net.mvpp2");
2809         if (mrvl_logtype >= 0)
2810                 rte_log_set_level(mrvl_logtype, RTE_LOG_NOTICE);
2811 }