drivers/net/mvpp2/mrvl_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Marvell International Ltd.
   3  * Copyright(c) 2017 Semihalf.
   4  * All rights reserved.
   5  */
   6
   7 #include <rte_ethdev_driver.h>
   8 #include <rte_kvargs.h>
   9 #include <rte_log.h>
  10 #include <rte_malloc.h>
  11 #include <rte_bus_vdev.h>
  12
  13 #include <fcntl.h>
  14 #include <linux/ethtool.h>
  15 #include <linux/sockios.h>
  16 #include <net/if.h>
  17 #include <net/if_arp.h>
  18 #include <sys/ioctl.h>
  19 #include <sys/socket.h>
  20 #include <sys/stat.h>
  21 #include <sys/types.h>
  22
  23 #include <rte_mvep_common.h>
  24 #include "mrvl_ethdev.h"
  25 #include "mrvl_qos.h"
  26 #include "mrvl_flow.h"
  27 #include "mrvl_mtr.h"
  28 #include "mrvl_tm.h"
  29
  30 /* bitmask with reserved hifs */
  31 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  32 /* bitmask with reserved bpools */
  33 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  34 /* bitmask with reserved kernel RSS tables */
  35 #define MRVL_MUSDK_RSS_RESERVED 0x01
  36 /* maximum number of available hifs */
  37 #define MRVL_MUSDK_HIFS_MAX 9
  38
  39 /* prefetch shift */
  40 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  41
  42 /* TCAM has 25 entries reserved for uc/mc filter entries */
  43 #define MRVL_MAC_ADDRS_MAX 25
  44 #define MRVL_MATCH_LEN 16
  45 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  46 /* Maximum allowable packet size */
  47 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  48
  49 #define MRVL_IFACE_NAME_ARG "iface"
  50 #define MRVL_CFG_ARG "cfg"
  51
  52 #define MRVL_BURST_SIZE 64
  53
  54 #define MRVL_ARP_LENGTH 28
  55
  56 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  57 #define MRVL_COOKIE_HIGH_ADDR_MASK 0xffffff0000000000
  58
  59 /** Port Rx offload capabilities */
  60 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
  61                           DEV_RX_OFFLOAD_JUMBO_FRAME | \
  62                           DEV_RX_OFFLOAD_CHECKSUM)
  63
  64 /** Port Tx offloads capabilities */
  65 #define MRVL_TX_OFFLOADS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
  66                           DEV_TX_OFFLOAD_UDP_CKSUM | \
  67                           DEV_TX_OFFLOAD_TCP_CKSUM)
  68
  69 static const char * const valid_args[] = {
  70         MRVL_IFACE_NAME_ARG,
  71         MRVL_CFG_ARG,
  72         NULL
  73 };
  74
  75 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
  76 static struct pp2_hif *hifs[RTE_MAX_LCORE];
  77 static int used_bpools[PP2_NUM_PKT_PROC] = {
  78         [0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED
  79 };
  80
  81 static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
  82 static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
  83 static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
  84
  85 int mrvl_logtype;
  86
  87 struct mrvl_ifnames {
  88         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
  89         int idx;
  90 };
  91
  92 /*
  93  * To use buffer harvesting based on loopback port shadow queue structure
  94  * was introduced for buffers information bookkeeping.
  95  *
  96  * Before sending the packet, related buffer information (pp2_buff_inf) is
  97  * stored in shadow queue. After packet is transmitted no longer used
  98  * packet buffer is released back to it's original hardware pool,
  99  * on condition it originated from interface.
 100  * In case it  was generated by application itself i.e: mbuf->port field is
 101  * 0xff then its released to software mempool.
 102  */
 103 struct mrvl_shadow_txq {
 104         int head;           /* write index - used when sending buffers */
 105         int tail;           /* read index - used when releasing buffers */
 106         u16 size;           /* queue occupied size */
 107         u16 num_to_release; /* number of buffers sent, that can be released */
 108         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 109 };
 110
 111 struct mrvl_rxq {
 112         struct mrvl_priv *priv;
 113         struct rte_mempool *mp;
 114         int queue_id;
 115         int port_id;
 116         int cksum_enabled;
 117         uint64_t bytes_recv;
 118         uint64_t drop_mac;
 119 };
 120
 121 struct mrvl_txq {
 122         struct mrvl_priv *priv;
 123         int queue_id;
 124         int port_id;
 125         uint64_t bytes_sent;
 126         struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE];
 127         int tx_deferred_start;
 128 };
 129
 130 static int mrvl_lcore_first;
 131 static int mrvl_lcore_last;
 132 static int mrvl_dev_num;
 133
 134 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
 135 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
 136                         struct pp2_hif *hif, unsigned int core_id,
 137                         struct mrvl_shadow_txq *sq, int qid, int force);
 138
 139 #define MRVL_XSTATS_TBL_ENTRY(name) { \
 140         #name, offsetof(struct pp2_ppio_statistics, name),      \
 141         sizeof(((struct pp2_ppio_statistics *)0)->name)         \
 142 }
 143
 144 /* Table with xstats data */
 145 static struct {
 146         const char *name;
 147         unsigned int offset;
 148         unsigned int size;
 149 } mrvl_xstats_tbl[] = {
 150         MRVL_XSTATS_TBL_ENTRY(rx_bytes),
 151         MRVL_XSTATS_TBL_ENTRY(rx_packets),
 152         MRVL_XSTATS_TBL_ENTRY(rx_unicast_packets),
 153         MRVL_XSTATS_TBL_ENTRY(rx_errors),
 154         MRVL_XSTATS_TBL_ENTRY(rx_fullq_dropped),
 155         MRVL_XSTATS_TBL_ENTRY(rx_bm_dropped),
 156         MRVL_XSTATS_TBL_ENTRY(rx_early_dropped),
 157         MRVL_XSTATS_TBL_ENTRY(rx_fifo_dropped),
 158         MRVL_XSTATS_TBL_ENTRY(rx_cls_dropped),
 159         MRVL_XSTATS_TBL_ENTRY(tx_bytes),
 160         MRVL_XSTATS_TBL_ENTRY(tx_packets),
 161         MRVL_XSTATS_TBL_ENTRY(tx_unicast_packets),
 162         MRVL_XSTATS_TBL_ENTRY(tx_errors)
 163 };
 164
 165 static inline int
 166 mrvl_get_bpool_size(int pp2_id, int pool_id)
 167 {
 168         int i;
 169         int size = 0;
 170
 171         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 172                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 173
 174         return size;
 175 }
 176
 177 static inline int
 178 mrvl_reserve_bit(int *bitmap, int max)
 179 {
 180         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 181
 182         if (n >= max)
 183                 return -1;
 184
 185         *bitmap |= 1 << n;
 186
 187         return n;
 188 }
 189
 190 static int
 191 mrvl_init_hif(int core_id)
 192 {
 193         struct pp2_hif_params params;
 194         char match[MRVL_MATCH_LEN];
 195         int ret;
 196
 197         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
 198         if (ret < 0) {
 199                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 200                 return ret;
 201         }
 202
 203         snprintf(match, sizeof(match), "hif-%d", ret);
 204         memset(&params, 0, sizeof(params));
 205         params.match = match;
 206         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
 207         ret = pp2_hif_init(&params, &hifs[core_id]);
 208         if (ret) {
 209                 MRVL_LOG(ERR, "Failed to initialize hif %d", core_id);
 210                 return ret;
 211         }
 212
 213         return 0;
 214 }
 215
 216 static inline struct pp2_hif*
 217 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
 218 {
 219         int ret;
 220
 221         if (likely(hifs[core_id] != NULL))
 222                 return hifs[core_id];
 223
 224         rte_spinlock_lock(&priv->lock);
 225
 226         ret = mrvl_init_hif(core_id);
 227         if (ret < 0) {
 228                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 229                 goto out;
 230         }
 231
 232         if (core_id < mrvl_lcore_first)
 233                 mrvl_lcore_first = core_id;
 234
 235         if (core_id > mrvl_lcore_last)
 236                 mrvl_lcore_last = core_id;
 237 out:
 238         rte_spinlock_unlock(&priv->lock);
 239
 240         return hifs[core_id];
 241 }
 242
 243 /**
 244  * Configure rss based on dpdk rss configuration.
 245  *
 246  * @param priv
 247  *   Pointer to private structure.
 248  * @param rss_conf
 249  *   Pointer to RSS configuration.
 250  *
 251  * @return
 252  *   0 on success, negative error value otherwise.
 253  */
 254 static int
 255 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 256 {
 257         if (rss_conf->rss_key)
 258                 MRVL_LOG(WARNING, "Changing hash key is not supported");
 259
 260         if (rss_conf->rss_hf == 0) {
 261                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 262         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 263                 priv->ppio_params.inqs_params.hash_type =
 264                         PP2_PPIO_HASH_T_2_TUPLE;
 265         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 266                 priv->ppio_params.inqs_params.hash_type =
 267                         PP2_PPIO_HASH_T_5_TUPLE;
 268                 priv->rss_hf_tcp = 1;
 269         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 270                 priv->ppio_params.inqs_params.hash_type =
 271                         PP2_PPIO_HASH_T_5_TUPLE;
 272                 priv->rss_hf_tcp = 0;
 273         } else {
 274                 return -EINVAL;
 275         }
 276
 277         return 0;
 278 }
 279
 280 /**
 281  * Ethernet device configuration.
 282  *
 283  * Prepare the driver for a given number of TX and RX queues and
 284  * configure RSS.
 285  *
 286  * @param dev
 287  *   Pointer to Ethernet device structure.
 288  *
 289  * @return
 290  *   0 on success, negative error value otherwise.
 291  */
 292 static int
 293 mrvl_dev_configure(struct rte_eth_dev *dev)
 294 {
 295         struct mrvl_priv *priv = dev->data->dev_private;
 296         int ret;
 297
 298         if (priv->ppio) {
 299                 MRVL_LOG(INFO, "Device reconfiguration is not supported");
 300                 return -EINVAL;
 301         }
 302
 303         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 304             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 305                 MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
 306                         dev->data->dev_conf.rxmode.mq_mode);
 307                 return -EINVAL;
 308         }
 309
 310         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 311                 MRVL_LOG(INFO, "Split headers not supported");
 312                 return -EINVAL;
 313         }
 314
 315         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 316                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 317                                  MRVL_PP2_ETH_HDRS_LEN;
 318
 319         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 320                                   dev->data->nb_rx_queues);
 321         if (ret < 0)
 322                 return ret;
 323
 324         ret = mrvl_configure_txqs(priv, dev->data->port_id,
 325                                   dev->data->nb_tx_queues);
 326         if (ret < 0)
 327                 return ret;
 328
 329         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 330         priv->ppio_params.maintain_stats = 1;
 331         priv->nb_rx_queues = dev->data->nb_rx_queues;
 332
 333         ret = mrvl_tm_init(dev);
 334         if (ret < 0)
 335                 return ret;
 336
 337         if (dev->data->nb_rx_queues == 1 &&
 338             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 339                 MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
 340                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 341
 342                 return 0;
 343         }
 344
 345         return mrvl_configure_rss(priv,
 346                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
 347 }
 348
 349 /**
 350  * DPDK callback to change the MTU.
 351  *
 352  * Setting the MTU affects hardware MRU (packets larger than the MRU
 353  * will be dropped).
 354  *
 355  * @param dev
 356  *   Pointer to Ethernet device structure.
 357  * @param mtu
 358  *   New MTU.
 359  *
 360  * @return
 361  *   0 on success, negative error value otherwise.
 362  */
 363 static int
 364 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 365 {
 366         struct mrvl_priv *priv = dev->data->dev_private;
 367         uint16_t mru;
 368         uint16_t mbuf_data_size = 0; /* SW buffer size */
 369         int ret;
 370
 371         mru = MRVL_PP2_MTU_TO_MRU(mtu);
 372         /*
 373          * min_rx_buf_size is equal to mbuf data size
 374          * if pmd didn't set it differently
 375          */
 376         mbuf_data_size = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
 377         /* Prevent PMD from:
 378          * - setting mru greater than the mbuf size resulting in
 379          * hw and sw buffer size mismatch
 380          * - setting mtu that requires the support of scattered packets
 381          * when this feature has not been enabled/supported so far
 382          * (TODO check scattered_rx flag here once scattered RX is supported).
 383          */
 384         if (mru + MRVL_PKT_OFFS > mbuf_data_size) {
 385                 mru = mbuf_data_size - MRVL_PKT_OFFS;
 386                 mtu = MRVL_PP2_MRU_TO_MTU(mru);
 387                 MRVL_LOG(WARNING, "MTU too big, max MTU possible limitted "
 388                         "by current mbuf size: %u. Set MTU to %u, MRU to %u",
 389                         mbuf_data_size, mtu, mru);
 390         }
 391
 392         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX) {
 393                 MRVL_LOG(ERR, "Invalid MTU [%u] or MRU [%u]", mtu, mru);
 394                 return -EINVAL;
 395         }
 396
 397         dev->data->mtu = mtu;
 398         dev->data->dev_conf.rxmode.max_rx_pkt_len = mru - MV_MH_SIZE;
 399
 400         if (!priv->ppio)
 401                 return 0;
 402
 403         ret = pp2_ppio_set_mru(priv->ppio, mru);
 404         if (ret) {
 405                 MRVL_LOG(ERR, "Failed to change MRU");
 406                 return ret;
 407         }
 408
 409         ret = pp2_ppio_set_mtu(priv->ppio, mtu);
 410         if (ret) {
 411                 MRVL_LOG(ERR, "Failed to change MTU");
 412                 return ret;
 413         }
 414
 415         return 0;
 416 }
 417
 418 /**
 419  * DPDK callback to bring the link up.
 420  *
 421  * @param dev
 422  *   Pointer to Ethernet device structure.
 423  *
 424  * @return
 425  *   0 on success, negative error value otherwise.
 426  */
 427 static int
 428 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 429 {
 430         struct mrvl_priv *priv = dev->data->dev_private;
 431         int ret;
 432
 433         if (!priv->ppio)
 434                 return -EPERM;
 435
 436         ret = pp2_ppio_enable(priv->ppio);
 437         if (ret)
 438                 return ret;
 439
 440         /*
 441          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 442          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 443          * PP2_TRAFFIC_INGRESS_EGRESS.
 444          *
 445          * Set mtu to default DPDK value here.
 446          */
 447         ret = mrvl_mtu_set(dev, dev->data->mtu);
 448         if (ret)
 449                 pp2_ppio_disable(priv->ppio);
 450
 451         return ret;
 452 }
 453
 454 /**
 455  * DPDK callback to bring the link down.
 456  *
 457  * @param dev
 458  *   Pointer to Ethernet device structure.
 459  *
 460  * @return
 461  *   0 on success, negative error value otherwise.
 462  */
 463 static int
 464 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 465 {
 466         struct mrvl_priv *priv = dev->data->dev_private;
 467
 468         if (!priv->ppio)
 469                 return -EPERM;
 470
 471         return pp2_ppio_disable(priv->ppio);
 472 }
 473
 474 /**
 475  * DPDK callback to start tx queue.
 476  *
 477  * @param dev
 478  *   Pointer to Ethernet device structure.
 479  * @param queue_id
 480  *   Transmit queue index.
 481  *
 482  * @return
 483  *   0 on success, negative error value otherwise.
 484  */
 485 static int
 486 mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
 487 {
 488         struct mrvl_priv *priv = dev->data->dev_private;
 489         int ret;
 490
 491         if (!priv)
 492                 return -EPERM;
 493
 494         /* passing 1 enables given tx queue */
 495         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1);
 496         if (ret) {
 497                 MRVL_LOG(ERR, "Failed to start txq %d", queue_id);
 498                 return ret;
 499         }
 500
 501         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 502
 503         return 0;
 504 }
 505
 506 /**
 507  * DPDK callback to stop tx queue.
 508  *
 509  * @param dev
 510  *   Pointer to Ethernet device structure.
 511  * @param queue_id
 512  *   Transmit queue index.
 513  *
 514  * @return
 515  *   0 on success, negative error value otherwise.
 516  */
 517 static int
 518 mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
 519 {
 520         struct mrvl_priv *priv = dev->data->dev_private;
 521         int ret;
 522
 523         if (!priv->ppio)
 524                 return -EPERM;
 525
 526         /* passing 0 disables given tx queue */
 527         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0);
 528         if (ret) {
 529                 MRVL_LOG(ERR, "Failed to stop txq %d", queue_id);
 530                 return ret;
 531         }
 532
 533         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 534
 535         return 0;
 536 }
 537
 538 /**
 539  * DPDK callback to start the device.
 540  *
 541  * @param dev
 542  *   Pointer to Ethernet device structure.
 543  *
 544  * @return
 545  *   0 on success, negative errno value on failure.
 546  */
 547 static int
 548 mrvl_dev_start(struct rte_eth_dev *dev)
 549 {
 550         struct mrvl_priv *priv = dev->data->dev_private;
 551         char match[MRVL_MATCH_LEN];
 552         int ret = 0, i, def_init_size;
 553
 554         if (priv->ppio)
 555                 return mrvl_dev_set_link_up(dev);
 556
 557         snprintf(match, sizeof(match), "ppio-%d:%d",
 558                  priv->pp_id, priv->ppio_id);
 559         priv->ppio_params.match = match;
 560
 561         /*
 562          * Calculate the minimum bpool size for refill feature as follows:
 563          * 2 default burst sizes multiply by number of rx queues.
 564          * If the bpool size will be below this value, new buffers will
 565          * be added to the pool.
 566          */
 567         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 568
 569         /* In case initial bpool size configured in queues setup is
 570          * smaller than minimum size add more buffers
 571          */
 572         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
 573         if (priv->bpool_init_size < def_init_size) {
 574                 int buffs_to_add = def_init_size - priv->bpool_init_size;
 575
 576                 priv->bpool_init_size += buffs_to_add;
 577                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
 578                 if (ret)
 579                         MRVL_LOG(ERR, "Failed to add buffers to bpool");
 580         }
 581
 582         /*
 583          * Calculate the maximum bpool size for refill feature as follows:
 584          * maximum number of descriptors in rx queue multiply by number
 585          * of rx queues plus minimum bpool size.
 586          * In case the bpool size will exceed this value, superfluous buffers
 587          * will be removed
 588          */
 589         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
 590                                 priv->bpool_min_size;
 591
 592         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 593         if (ret) {
 594                 MRVL_LOG(ERR, "Failed to init ppio");
 595                 return ret;
 596         }
 597
 598         /*
 599          * In case there are some some stale uc/mc mac addresses flush them
 600          * here. It cannot be done during mrvl_dev_close() as port information
 601          * is already gone at that point (due to pp2_ppio_deinit() in
 602          * mrvl_dev_stop()).
 603          */
 604         if (!priv->uc_mc_flushed) {
 605                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 606                 if (ret) {
 607                         MRVL_LOG(ERR,
 608                                 "Failed to flush uc/mc filter list");
 609                         goto out;
 610                 }
 611                 priv->uc_mc_flushed = 1;
 612         }
 613
 614         if (!priv->vlan_flushed) {
 615                 ret = pp2_ppio_flush_vlan(priv->ppio);
 616                 if (ret) {
 617                         MRVL_LOG(ERR, "Failed to flush vlan list");
 618                         /*
 619                          * TODO
 620                          * once pp2_ppio_flush_vlan() is supported jump to out
 621                          * goto out;
 622                          */
 623                 }
 624                 priv->vlan_flushed = 1;
 625         }
 626         ret = mrvl_mtu_set(dev, dev->data->mtu);
 627         if (ret)
 628                 MRVL_LOG(ERR, "Failed to set MTU to %d", dev->data->mtu);
 629
 630         /* For default QoS config, don't start classifier. */
 631         if (mrvl_qos_cfg  &&
 632             mrvl_qos_cfg->port[dev->data->port_id].use_global_defaults == 0) {
 633                 ret = mrvl_start_qos_mapping(priv);
 634                 if (ret) {
 635                         MRVL_LOG(ERR, "Failed to setup QoS mapping");
 636                         goto out;
 637                 }
 638         }
 639
 640         ret = mrvl_dev_set_link_up(dev);
 641         if (ret) {
 642                 MRVL_LOG(ERR, "Failed to set link up");
 643                 goto out;
 644         }
 645
 646         /* start tx queues */
 647         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 648                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 649
 650                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
 651
 652                 if (!txq->tx_deferred_start)
 653                         continue;
 654
 655                 /*
 656                  * All txqs are started by default. Stop them
 657                  * so that tx_deferred_start works as expected.
 658                  */
 659                 ret = mrvl_tx_queue_stop(dev, i);
 660                 if (ret)
 661                         goto out;
 662         }
 663
 664         mrvl_flow_init(dev);
 665         mrvl_mtr_init(dev);
 666
 667         return 0;
 668 out:
 669         MRVL_LOG(ERR, "Failed to start device");
 670         pp2_ppio_deinit(priv->ppio);
 671         return ret;
 672 }
 673
 674 /**
 675  * Flush receive queues.
 676  *
 677  * @param dev
 678  *   Pointer to Ethernet device structure.
 679  */
 680 static void
 681 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 682 {
 683         int i;
 684
 685         MRVL_LOG(INFO, "Flushing rx queues");
 686         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 687                 int ret, num;
 688
 689                 do {
 690                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 691                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 692
 693                         num = MRVL_PP2_RXD_MAX;
 694                         ret = pp2_ppio_recv(q->priv->ppio,
 695                                             q->priv->rxq_map[q->queue_id].tc,
 696                                             q->priv->rxq_map[q->queue_id].inq,
 697                                             descs, (uint16_t *)&num);
 698                 } while (ret == 0 && num);
 699         }
 700 }
 701
 702 /**
 703  * Flush transmit shadow queues.
 704  *
 705  * @param dev
 706  *   Pointer to Ethernet device structure.
 707  */
 708 static void
 709 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 710 {
 711         int i, j;
 712         struct mrvl_txq *txq;
 713
 714         MRVL_LOG(INFO, "Flushing tx shadow queues");
 715         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 716                 txq = (struct mrvl_txq *)dev->data->tx_queues[i];
 717
 718                 for (j = 0; j < RTE_MAX_LCORE; j++) {
 719                         struct mrvl_shadow_txq *sq;
 720
 721                         if (!hifs[j])
 722                                 continue;
 723
 724                         sq = &txq->shadow_txqs[j];
 725                         mrvl_free_sent_buffers(txq->priv->ppio,
 726                                 hifs[j], j, sq, txq->queue_id, 1);
 727                         while (sq->tail != sq->head) {
 728                                 uint64_t addr = cookie_addr_high |
 729                                         sq->ent[sq->tail].buff.cookie;
 730                                 rte_pktmbuf_free(
 731                                         (struct rte_mbuf *)addr);
 732                                 sq->tail = (sq->tail + 1) &
 733                                             MRVL_PP2_TX_SHADOWQ_MASK;
 734                         }
 735                         memset(sq, 0, sizeof(*sq));
 736                 }
 737         }
 738 }
 739
 740 /**
 741  * Flush hardware bpool (buffer-pool).
 742  *
 743  * @param dev
 744  *   Pointer to Ethernet device structure.
 745  */
 746 static void
 747 mrvl_flush_bpool(struct rte_eth_dev *dev)
 748 {
 749         struct mrvl_priv *priv = dev->data->dev_private;
 750         struct pp2_hif *hif;
 751         uint32_t num;
 752         int ret;
 753         unsigned int core_id = rte_lcore_id();
 754
 755         if (core_id == LCORE_ID_ANY)
 756                 core_id = 0;
 757
 758         hif = mrvl_get_hif(priv, core_id);
 759
 760         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 761         if (ret) {
 762                 MRVL_LOG(ERR, "Failed to get bpool buffers number");
 763                 return;
 764         }
 765
 766         while (num--) {
 767                 struct pp2_buff_inf inf;
 768                 uint64_t addr;
 769
 770                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
 771                 if (ret)
 772                         break;
 773
 774                 addr = cookie_addr_high | inf.cookie;
 775                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 776         }
 777 }
 778
 779 /**
 780  * DPDK callback to stop the device.
 781  *
 782  * @param dev
 783  *   Pointer to Ethernet device structure.
 784  */
 785 static void
 786 mrvl_dev_stop(struct rte_eth_dev *dev)
 787 {
 788         mrvl_dev_set_link_down(dev);
 789 }
 790
 791 /**
 792  * DPDK callback to close the device.
 793  *
 794  * @param dev
 795  *   Pointer to Ethernet device structure.
 796  */
 797 static void
 798 mrvl_dev_close(struct rte_eth_dev *dev)
 799 {
 800         struct mrvl_priv *priv = dev->data->dev_private;
 801         size_t i;
 802
 803         mrvl_flush_rx_queues(dev);
 804         mrvl_flush_tx_shadow_queues(dev);
 805         mrvl_flow_deinit(dev);
 806         mrvl_mtr_deinit(dev);
 807
 808         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 809                 struct pp2_ppio_tc_params *tc_params =
 810                         &priv->ppio_params.inqs_params.tcs_params[i];
 811
 812                 if (tc_params->inqs_params) {
 813                         rte_free(tc_params->inqs_params);
 814                         tc_params->inqs_params = NULL;
 815                 }
 816         }
 817
 818         if (priv->cls_tbl) {
 819                 pp2_cls_tbl_deinit(priv->cls_tbl);
 820                 priv->cls_tbl = NULL;
 821         }
 822
 823         if (priv->qos_tbl) {
 824                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 825                 priv->qos_tbl = NULL;
 826         }
 827
 828         mrvl_flush_bpool(dev);
 829         mrvl_tm_deinit(dev);
 830
 831         if (priv->ppio) {
 832                 pp2_ppio_deinit(priv->ppio);
 833                 priv->ppio = NULL;
 834         }
 835
 836         /* policer must be released after ppio deinitialization */
 837         if (priv->default_policer) {
 838                 pp2_cls_plcr_deinit(priv->default_policer);
 839                 priv->default_policer = NULL;
 840         }
 841 }
 842
 843 /**
 844  * DPDK callback to retrieve physical link information.
 845  *
 846  * @param dev
 847  *   Pointer to Ethernet device structure.
 848  * @param wait_to_complete
 849  *   Wait for request completion (ignored).
 850  *
 851  * @return
 852  *   0 on success, negative error value otherwise.
 853  */
 854 static int
 855 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 856 {
 857         /*
 858          * TODO
 859          * once MUSDK provides necessary API use it here
 860          */
 861         struct mrvl_priv *priv = dev->data->dev_private;
 862         struct ethtool_cmd edata;
 863         struct ifreq req;
 864         int ret, fd, link_up;
 865
 866         if (!priv->ppio)
 867                 return -EPERM;
 868
 869         edata.cmd = ETHTOOL_GSET;
 870
 871         strcpy(req.ifr_name, dev->data->name);
 872         req.ifr_data = (void *)&edata;
 873
 874         fd = socket(AF_INET, SOCK_DGRAM, 0);
 875         if (fd == -1)
 876                 return -EFAULT;
 877
 878         ret = ioctl(fd, SIOCETHTOOL, &req);
 879         if (ret == -1) {
 880                 close(fd);
 881                 return -EFAULT;
 882         }
 883
 884         close(fd);
 885
 886         switch (ethtool_cmd_speed(&edata)) {
 887         case SPEED_10:
 888                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 889                 break;
 890         case SPEED_100:
 891                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 892                 break;
 893         case SPEED_1000:
 894                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 895                 break;
 896         case SPEED_10000:
 897                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 898                 break;
 899         default:
 900                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 901         }
 902
 903         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 904                                                          ETH_LINK_HALF_DUPLEX;
 905         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 906                                                            ETH_LINK_FIXED;
 907         pp2_ppio_get_link_state(priv->ppio, &link_up);
 908         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
 909
 910         return 0;
 911 }
 912
 913 /**
 914  * DPDK callback to enable promiscuous mode.
 915  *
 916  * @param dev
 917  *   Pointer to Ethernet device structure.
 918  */
 919 static void
 920 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
 921 {
 922         struct mrvl_priv *priv = dev->data->dev_private;
 923         int ret;
 924
 925         if (!priv->ppio)
 926                 return;
 927
 928         if (priv->isolated)
 929                 return;
 930
 931         ret = pp2_ppio_set_promisc(priv->ppio, 1);
 932         if (ret)
 933                 MRVL_LOG(ERR, "Failed to enable promiscuous mode");
 934 }
 935
 936 /**
 937  * DPDK callback to enable allmulti mode.
 938  *
 939  * @param dev
 940  *   Pointer to Ethernet device structure.
 941  */
 942 static void
 943 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
 944 {
 945         struct mrvl_priv *priv = dev->data->dev_private;
 946         int ret;
 947
 948         if (!priv->ppio)
 949                 return;
 950
 951         if (priv->isolated)
 952                 return;
 953
 954         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
 955         if (ret)
 956                 MRVL_LOG(ERR, "Failed enable all-multicast mode");
 957 }
 958
 959 /**
 960  * DPDK callback to disable promiscuous mode.
 961  *
 962  * @param dev
 963  *   Pointer to Ethernet device structure.
 964  */
 965 static void
 966 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
 967 {
 968         struct mrvl_priv *priv = dev->data->dev_private;
 969         int ret;
 970
 971         if (!priv->ppio)
 972                 return;
 973
 974         ret = pp2_ppio_set_promisc(priv->ppio, 0);
 975         if (ret)
 976                 MRVL_LOG(ERR, "Failed to disable promiscuous mode");
 977 }
 978
 979 /**
 980  * DPDK callback to disable allmulticast mode.
 981  *
 982  * @param dev
 983  *   Pointer to Ethernet device structure.
 984  */
 985 static void
 986 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
 987 {
 988         struct mrvl_priv *priv = dev->data->dev_private;
 989         int ret;
 990
 991         if (!priv->ppio)
 992                 return;
 993
 994         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
 995         if (ret)
 996                 MRVL_LOG(ERR, "Failed to disable all-multicast mode");
 997 }
 998
 999 /**
1000  * DPDK callback to remove a MAC address.
1001  *
1002  * @param dev
1003  *   Pointer to Ethernet device structure.
1004  * @param index
1005  *   MAC address index.
1006  */
1007 static void
1008 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1009 {
1010         struct mrvl_priv *priv = dev->data->dev_private;
1011         char buf[ETHER_ADDR_FMT_SIZE];
1012         int ret;
1013
1014         if (!priv->ppio)
1015                 return;
1016
1017         if (priv->isolated)
1018                 return;
1019
1020         ret = pp2_ppio_remove_mac_addr(priv->ppio,
1021                                        dev->data->mac_addrs[index].addr_bytes);
1022         if (ret) {
1023                 ether_format_addr(buf, sizeof(buf),
1024                                   &dev->data->mac_addrs[index]);
1025                 MRVL_LOG(ERR, "Failed to remove mac %s", buf);
1026         }
1027 }
1028
1029 /**
1030  * DPDK callback to add a MAC address.
1031  *
1032  * @param dev
1033  *   Pointer to Ethernet device structure.
1034  * @param mac_addr
1035  *   MAC address to register.
1036  * @param index
1037  *   MAC address index.
1038  * @param vmdq
1039  *   VMDq pool index to associate address with (unused).
1040  *
1041  * @return
1042  *   0 on success, negative error value otherwise.
1043  */
1044 static int
1045 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
1046                   uint32_t index, uint32_t vmdq __rte_unused)
1047 {
1048         struct mrvl_priv *priv = dev->data->dev_private;
1049         char buf[ETHER_ADDR_FMT_SIZE];
1050         int ret;
1051
1052         if (priv->isolated)
1053                 return -ENOTSUP;
1054
1055         if (index == 0)
1056                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
1057                 return -1;
1058
1059         if (!priv->ppio)
1060                 return 0;
1061
1062         /*
1063          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
1064          * parameter uc_filter_max. Maximum number of mc addresses is then
1065          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
1066          * 21 respectively.
1067          *
1068          * If more than uc_filter_max uc addresses were added to filter list
1069          * then NIC will switch to promiscuous mode automatically.
1070          *
1071          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
1072          * were added to filter list then NIC will switch to all-multicast mode
1073          * automatically.
1074          */
1075         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
1076         if (ret) {
1077                 ether_format_addr(buf, sizeof(buf), mac_addr);
1078                 MRVL_LOG(ERR, "Failed to add mac %s", buf);
1079                 return -1;
1080         }
1081
1082         return 0;
1083 }
1084
1085 /**
1086  * DPDK callback to set the primary MAC address.
1087  *
1088  * @param dev
1089  *   Pointer to Ethernet device structure.
1090  * @param mac_addr
1091  *   MAC address to register.
1092  *
1093  * @return
1094  *   0 on success, negative error value otherwise.
1095  */
1096 static int
1097 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1098 {
1099         struct mrvl_priv *priv = dev->data->dev_private;
1100         int ret;
1101
1102         if (!priv->ppio)
1103                 return 0;
1104
1105         if (priv->isolated)
1106                 return -ENOTSUP;
1107
1108         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
1109         if (ret) {
1110                 char buf[ETHER_ADDR_FMT_SIZE];
1111                 ether_format_addr(buf, sizeof(buf), mac_addr);
1112                 MRVL_LOG(ERR, "Failed to set mac to %s", buf);
1113         }
1114
1115         return ret;
1116 }
1117
1118 /**
1119  * DPDK callback to get device statistics.
1120  *
1121  * @param dev
1122  *   Pointer to Ethernet device structure.
1123  * @param stats
1124  *   Stats structure output buffer.
1125  *
1126  * @return
1127  *   0 on success, negative error value otherwise.
1128  */
1129 static int
1130 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1131 {
1132         struct mrvl_priv *priv = dev->data->dev_private;
1133         struct pp2_ppio_statistics ppio_stats;
1134         uint64_t drop_mac = 0;
1135         unsigned int i, idx, ret;
1136
1137         if (!priv->ppio)
1138                 return -EPERM;
1139
1140         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1141                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1142                 struct pp2_ppio_inq_statistics rx_stats;
1143
1144                 if (!rxq)
1145                         continue;
1146
1147                 idx = rxq->queue_id;
1148                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1149                         MRVL_LOG(ERR,
1150                                 "rx queue %d stats out of range (0 - %d)",
1151                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1152                         continue;
1153                 }
1154
1155                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
1156                                                   priv->rxq_map[idx].tc,
1157                                                   priv->rxq_map[idx].inq,
1158                                                   &rx_stats, 0);
1159                 if (unlikely(ret)) {
1160                         MRVL_LOG(ERR,
1161                                 "Failed to update rx queue %d stats", idx);
1162                         break;
1163                 }
1164
1165                 stats->q_ibytes[idx] = rxq->bytes_recv;
1166                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
1167                 stats->q_errors[idx] = rx_stats.drop_early +
1168                                        rx_stats.drop_fullq +
1169                                        rx_stats.drop_bm +
1170                                        rxq->drop_mac;
1171                 stats->ibytes += rxq->bytes_recv;
1172                 drop_mac += rxq->drop_mac;
1173         }
1174
1175         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1176                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1177                 struct pp2_ppio_outq_statistics tx_stats;
1178
1179                 if (!txq)
1180                         continue;
1181
1182                 idx = txq->queue_id;
1183                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1184                         MRVL_LOG(ERR,
1185                                 "tx queue %d stats out of range (0 - %d)",
1186                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1187                 }
1188
1189                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1190                                                    &tx_stats, 0);
1191                 if (unlikely(ret)) {
1192                         MRVL_LOG(ERR,
1193                                 "Failed to update tx queue %d stats", idx);
1194                         break;
1195                 }
1196
1197                 stats->q_opackets[idx] = tx_stats.deq_desc;
1198                 stats->q_obytes[idx] = txq->bytes_sent;
1199                 stats->obytes += txq->bytes_sent;
1200         }
1201
1202         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1203         if (unlikely(ret)) {
1204                 MRVL_LOG(ERR, "Failed to update port statistics");
1205                 return ret;
1206         }
1207
1208         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1209         stats->opackets += ppio_stats.tx_packets;
1210         stats->imissed += ppio_stats.rx_fullq_dropped +
1211                           ppio_stats.rx_bm_dropped +
1212                           ppio_stats.rx_early_dropped +
1213                           ppio_stats.rx_fifo_dropped +
1214                           ppio_stats.rx_cls_dropped;
1215         stats->ierrors = drop_mac;
1216
1217         return 0;
1218 }
1219
1220 /**
1221  * DPDK callback to clear device statistics.
1222  *
1223  * @param dev
1224  *   Pointer to Ethernet device structure.
1225  */
1226 static void
1227 mrvl_stats_reset(struct rte_eth_dev *dev)
1228 {
1229         struct mrvl_priv *priv = dev->data->dev_private;
1230         int i;
1231
1232         if (!priv->ppio)
1233                 return;
1234
1235         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1236                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1237
1238                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1239                                             priv->rxq_map[i].inq, NULL, 1);
1240                 rxq->bytes_recv = 0;
1241                 rxq->drop_mac = 0;
1242         }
1243
1244         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1245                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1246
1247                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1248                 txq->bytes_sent = 0;
1249         }
1250
1251         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1252 }
1253
1254 /**
1255  * DPDK callback to get extended statistics.
1256  *
1257  * @param dev
1258  *   Pointer to Ethernet device structure.
1259  * @param stats
1260  *   Pointer to xstats table.
1261  * @param n
1262  *   Number of entries in xstats table.
1263  * @return
1264  *   Negative value on error, number of read xstats otherwise.
1265  */
1266 static int
1267 mrvl_xstats_get(struct rte_eth_dev *dev,
1268                 struct rte_eth_xstat *stats, unsigned int n)
1269 {
1270         struct mrvl_priv *priv = dev->data->dev_private;
1271         struct pp2_ppio_statistics ppio_stats;
1272         unsigned int i;
1273
1274         if (!stats)
1275                 return 0;
1276
1277         pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1278         for (i = 0; i < n && i < RTE_DIM(mrvl_xstats_tbl); i++) {
1279                 uint64_t val;
1280
1281                 if (mrvl_xstats_tbl[i].size == sizeof(uint32_t))
1282                         val = *(uint32_t *)((uint8_t *)&ppio_stats +
1283                                             mrvl_xstats_tbl[i].offset);
1284                 else if (mrvl_xstats_tbl[i].size == sizeof(uint64_t))
1285                         val = *(uint64_t *)((uint8_t *)&ppio_stats +
1286                                             mrvl_xstats_tbl[i].offset);
1287                 else
1288                         return -EINVAL;
1289
1290                 stats[i].id = i;
1291                 stats[i].value = val;
1292         }
1293
1294         return n;
1295 }
1296
1297 /**
1298  * DPDK callback to reset extended statistics.
1299  *
1300  * @param dev
1301  *   Pointer to Ethernet device structure.
1302  */
1303 static void
1304 mrvl_xstats_reset(struct rte_eth_dev *dev)
1305 {
1306         mrvl_stats_reset(dev);
1307 }
1308
1309 /**
1310  * DPDK callback to get extended statistics names.
1311  *
1312  * @param dev (unused)
1313  *   Pointer to Ethernet device structure.
1314  * @param xstats_names
1315  *   Pointer to xstats names table.
1316  * @param size
1317  *   Size of the xstats names table.
1318  * @return
1319  *   Number of read names.
1320  */
1321 static int
1322 mrvl_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
1323                       struct rte_eth_xstat_name *xstats_names,
1324                       unsigned int size)
1325 {
1326         unsigned int i;
1327
1328         if (!xstats_names)
1329                 return RTE_DIM(mrvl_xstats_tbl);
1330
1331         for (i = 0; i < size && i < RTE_DIM(mrvl_xstats_tbl); i++)
1332                 snprintf(xstats_names[i].name, RTE_ETH_XSTATS_NAME_SIZE, "%s",
1333                          mrvl_xstats_tbl[i].name);
1334
1335         return size;
1336 }
1337
1338 /**
1339  * DPDK callback to get information about the device.
1340  *
1341  * @param dev
1342  *   Pointer to Ethernet device structure (unused).
1343  * @param info
1344  *   Info structure output buffer.
1345  */
1346 static void
1347 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
1348                    struct rte_eth_dev_info *info)
1349 {
1350         info->speed_capa = ETH_LINK_SPEED_10M |
1351                            ETH_LINK_SPEED_100M |
1352                            ETH_LINK_SPEED_1G |
1353                            ETH_LINK_SPEED_10G;
1354
1355         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1356         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1357         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1358
1359         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1360         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1361         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1362
1363         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1364         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1365         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1366
1367         info->rx_offload_capa = MRVL_RX_OFFLOADS;
1368         info->rx_queue_offload_capa = MRVL_RX_OFFLOADS;
1369
1370         info->tx_offload_capa = MRVL_TX_OFFLOADS;
1371         info->tx_queue_offload_capa = MRVL_TX_OFFLOADS;
1372
1373         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1374                                        ETH_RSS_NONFRAG_IPV4_TCP |
1375                                        ETH_RSS_NONFRAG_IPV4_UDP;
1376
1377         /* By default packets are dropped if no descriptors are available */
1378         info->default_rxconf.rx_drop_en = 1;
1379
1380         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1381 }
1382
1383 /**
1384  * Return supported packet types.
1385  *
1386  * @param dev
1387  *   Pointer to Ethernet device structure (unused).
1388  *
1389  * @return
1390  *   Const pointer to the table with supported packet types.
1391  */
1392 static const uint32_t *
1393 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1394 {
1395         static const uint32_t ptypes[] = {
1396                 RTE_PTYPE_L2_ETHER,
1397                 RTE_PTYPE_L2_ETHER_VLAN,
1398                 RTE_PTYPE_L2_ETHER_QINQ,
1399                 RTE_PTYPE_L3_IPV4,
1400                 RTE_PTYPE_L3_IPV4_EXT,
1401                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1402                 RTE_PTYPE_L3_IPV6,
1403                 RTE_PTYPE_L3_IPV6_EXT,
1404                 RTE_PTYPE_L2_ETHER_ARP,
1405                 RTE_PTYPE_L4_TCP,
1406                 RTE_PTYPE_L4_UDP
1407         };
1408
1409         return ptypes;
1410 }
1411
1412 /**
1413  * DPDK callback to get information about specific receive queue.
1414  *
1415  * @param dev
1416  *   Pointer to Ethernet device structure.
1417  * @param rx_queue_id
1418  *   Receive queue index.
1419  * @param qinfo
1420  *   Receive queue information structure.
1421  */
1422 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1423                               struct rte_eth_rxq_info *qinfo)
1424 {
1425         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1426         struct mrvl_priv *priv = dev->data->dev_private;
1427         int inq = priv->rxq_map[rx_queue_id].inq;
1428         int tc = priv->rxq_map[rx_queue_id].tc;
1429         struct pp2_ppio_tc_params *tc_params =
1430                 &priv->ppio_params.inqs_params.tcs_params[tc];
1431
1432         qinfo->mp = q->mp;
1433         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1434 }
1435
1436 /**
1437  * DPDK callback to get information about specific transmit queue.
1438  *
1439  * @param dev
1440  *   Pointer to Ethernet device structure.
1441  * @param tx_queue_id
1442  *   Transmit queue index.
1443  * @param qinfo
1444  *   Transmit queue information structure.
1445  */
1446 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1447                               struct rte_eth_txq_info *qinfo)
1448 {
1449         struct mrvl_priv *priv = dev->data->dev_private;
1450         struct mrvl_txq *txq = dev->data->tx_queues[tx_queue_id];
1451
1452         qinfo->nb_desc =
1453                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1454         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1455 }
1456
1457 /**
1458  * DPDK callback to Configure a VLAN filter.
1459  *
1460  * @param dev
1461  *   Pointer to Ethernet device structure.
1462  * @param vlan_id
1463  *   VLAN ID to filter.
1464  * @param on
1465  *   Toggle filter.
1466  *
1467  * @return
1468  *   0 on success, negative error value otherwise.
1469  */
1470 static int
1471 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1472 {
1473         struct mrvl_priv *priv = dev->data->dev_private;
1474
1475         if (!priv->ppio)
1476                 return -EPERM;
1477
1478         if (priv->isolated)
1479                 return -ENOTSUP;
1480
1481         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1482                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1483 }
1484
1485 /**
1486  * Release buffers to hardware bpool (buffer-pool)
1487  *
1488  * @param rxq
1489  *   Receive queue pointer.
1490  * @param num
1491  *   Number of buffers to release to bpool.
1492  *
1493  * @return
1494  *   0 on success, negative error value otherwise.
1495  */
1496 static int
1497 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1498 {
1499         struct buff_release_entry entries[MRVL_PP2_RXD_MAX];
1500         struct rte_mbuf *mbufs[MRVL_PP2_RXD_MAX];
1501         int i, ret;
1502         unsigned int core_id;
1503         struct pp2_hif *hif;
1504         struct pp2_bpool *bpool;
1505
1506         core_id = rte_lcore_id();
1507         if (core_id == LCORE_ID_ANY)
1508                 core_id = 0;
1509
1510         hif = mrvl_get_hif(rxq->priv, core_id);
1511         if (!hif)
1512                 return -1;
1513
1514         bpool = rxq->priv->bpool;
1515
1516         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1517         if (ret)
1518                 return ret;
1519
1520         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1521                 cookie_addr_high =
1522                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1523
1524         for (i = 0; i < num; i++) {
1525                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1526                         != cookie_addr_high) {
1527                         MRVL_LOG(ERR,
1528                                 "mbuf virtual addr high 0x%lx out of range",
1529                                 (uint64_t)mbufs[i] >> 32);
1530                         goto out;
1531                 }
1532
1533                 entries[i].buff.addr =
1534                         rte_mbuf_data_iova_default(mbufs[i]);
1535                 entries[i].buff.cookie = (uint64_t)mbufs[i];
1536                 entries[i].bpool = bpool;
1537         }
1538
1539         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1540         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1541
1542         if (i != num)
1543                 goto out;
1544
1545         return 0;
1546 out:
1547         for (; i < num; i++)
1548                 rte_pktmbuf_free(mbufs[i]);
1549
1550         return -1;
1551 }
1552
1553 /**
1554  * DPDK callback to configure the receive queue.
1555  *
1556  * @param dev
1557  *   Pointer to Ethernet device structure.
1558  * @param idx
1559  *   RX queue index.
1560  * @param desc
1561  *   Number of descriptors to configure in queue.
1562  * @param socket
1563  *   NUMA socket on which memory must be allocated.
1564  * @param conf
1565  *   Thresholds parameters.
1566  * @param mp
1567  *   Memory pool for buffer allocations.
1568  *
1569  * @return
1570  *   0 on success, negative error value otherwise.
1571  */
1572 static int
1573 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1574                     unsigned int socket,
1575                     const struct rte_eth_rxconf *conf,
1576                     struct rte_mempool *mp)
1577 {
1578         struct mrvl_priv *priv = dev->data->dev_private;
1579         struct mrvl_rxq *rxq;
1580         uint32_t frame_size, buf_size = rte_pktmbuf_data_room_size(mp);
1581         uint32_t max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1582         int ret, tc, inq;
1583         uint64_t offloads;
1584
1585         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1586
1587         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1588                 /*
1589                  * Unknown TC mapping, mapping will not have a correct queue.
1590                  */
1591                 MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu",
1592                         idx, priv->ppio_id);
1593                 return -EFAULT;
1594         }
1595
1596         frame_size = buf_size - RTE_PKTMBUF_HEADROOM - MRVL_PKT_EFFEC_OFFS;
1597         if (frame_size < max_rx_pkt_len) {
1598                 MRVL_LOG(WARNING,
1599                         "Mbuf size must be increased to %u bytes to hold up "
1600                         "to %u bytes of data.",
1601                         buf_size + max_rx_pkt_len - frame_size,
1602                         max_rx_pkt_len);
1603                 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
1604                 MRVL_LOG(INFO, "Setting max rx pkt len to %u",
1605                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
1606         }
1607
1608         if (dev->data->rx_queues[idx]) {
1609                 rte_free(dev->data->rx_queues[idx]);
1610                 dev->data->rx_queues[idx] = NULL;
1611         }
1612
1613         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1614         if (!rxq)
1615                 return -ENOMEM;
1616
1617         rxq->priv = priv;
1618         rxq->mp = mp;
1619         rxq->cksum_enabled = offloads & DEV_RX_OFFLOAD_IPV4_CKSUM;
1620         rxq->queue_id = idx;
1621         rxq->port_id = dev->data->port_id;
1622         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1623
1624         tc = priv->rxq_map[rxq->queue_id].tc,
1625         inq = priv->rxq_map[rxq->queue_id].inq;
1626         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1627                 desc;
1628
1629         ret = mrvl_fill_bpool(rxq, desc);
1630         if (ret) {
1631                 rte_free(rxq);
1632                 return ret;
1633         }
1634
1635         priv->bpool_init_size += desc;
1636
1637         dev->data->rx_queues[idx] = rxq;
1638
1639         return 0;
1640 }
1641
1642 /**
1643  * DPDK callback to release the receive queue.
1644  *
1645  * @param rxq
1646  *   Generic receive queue pointer.
1647  */
1648 static void
1649 mrvl_rx_queue_release(void *rxq)
1650 {
1651         struct mrvl_rxq *q = rxq;
1652         struct pp2_ppio_tc_params *tc_params;
1653         int i, num, tc, inq;
1654         struct pp2_hif *hif;
1655         unsigned int core_id = rte_lcore_id();
1656
1657         if (core_id == LCORE_ID_ANY)
1658                 core_id = 0;
1659
1660         if (!q)
1661                 return;
1662
1663         hif = mrvl_get_hif(q->priv, core_id);
1664
1665         if (!hif)
1666                 return;
1667
1668         tc = q->priv->rxq_map[q->queue_id].tc;
1669         inq = q->priv->rxq_map[q->queue_id].inq;
1670         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1671         num = tc_params->inqs_params[inq].size;
1672         for (i = 0; i < num; i++) {
1673                 struct pp2_buff_inf inf;
1674                 uint64_t addr;
1675
1676                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
1677                 addr = cookie_addr_high | inf.cookie;
1678                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1679         }
1680
1681         rte_free(q);
1682 }
1683
1684 /**
1685  * DPDK callback to configure the transmit queue.
1686  *
1687  * @param dev
1688  *   Pointer to Ethernet device structure.
1689  * @param idx
1690  *   Transmit queue index.
1691  * @param desc
1692  *   Number of descriptors to configure in the queue.
1693  * @param socket
1694  *   NUMA socket on which memory must be allocated.
1695  * @param conf
1696  *   Tx queue configuration parameters.
1697  *
1698  * @return
1699  *   0 on success, negative error value otherwise.
1700  */
1701 static int
1702 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1703                     unsigned int socket,
1704                     const struct rte_eth_txconf *conf)
1705 {
1706         struct mrvl_priv *priv = dev->data->dev_private;
1707         struct mrvl_txq *txq;
1708
1709         if (dev->data->tx_queues[idx]) {
1710                 rte_free(dev->data->tx_queues[idx]);
1711                 dev->data->tx_queues[idx] = NULL;
1712         }
1713
1714         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1715         if (!txq)
1716                 return -ENOMEM;
1717
1718         txq->priv = priv;
1719         txq->queue_id = idx;
1720         txq->port_id = dev->data->port_id;
1721         txq->tx_deferred_start = conf->tx_deferred_start;
1722         dev->data->tx_queues[idx] = txq;
1723
1724         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1725
1726         return 0;
1727 }
1728
1729 /**
1730  * DPDK callback to release the transmit queue.
1731  *
1732  * @param txq
1733  *   Generic transmit queue pointer.
1734  */
1735 static void
1736 mrvl_tx_queue_release(void *txq)
1737 {
1738         struct mrvl_txq *q = txq;
1739
1740         if (!q)
1741                 return;
1742
1743         rte_free(q);
1744 }
1745
1746 /**
1747  * DPDK callback to get flow control configuration.
1748  *
1749  * @param dev
1750  *  Pointer to Ethernet device structure.
1751  * @param fc_conf
1752  *  Pointer to the flow control configuration.
1753  *
1754  * @return
1755  *  0 on success, negative error value otherwise.
1756  */
1757 static int
1758 mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1759 {
1760         struct mrvl_priv *priv = dev->data->dev_private;
1761         int ret, en;
1762
1763         if (!priv)
1764                 return -EPERM;
1765
1766         ret = pp2_ppio_get_rx_pause(priv->ppio, &en);
1767         if (ret) {
1768                 MRVL_LOG(ERR, "Failed to read rx pause state");
1769                 return ret;
1770         }
1771
1772         fc_conf->mode = en ? RTE_FC_RX_PAUSE : RTE_FC_NONE;
1773
1774         return 0;
1775 }
1776
1777 /**
1778  * DPDK callback to set flow control configuration.
1779  *
1780  * @param dev
1781  *  Pointer to Ethernet device structure.
1782  * @param fc_conf
1783  *  Pointer to the flow control configuration.
1784  *
1785  * @return
1786  *  0 on success, negative error value otherwise.
1787  */
1788 static int
1789 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1790 {
1791         struct mrvl_priv *priv = dev->data->dev_private;
1792
1793         if (!priv)
1794                 return -EPERM;
1795
1796         if (fc_conf->high_water ||
1797             fc_conf->low_water ||
1798             fc_conf->pause_time ||
1799             fc_conf->mac_ctrl_frame_fwd ||
1800             fc_conf->autoneg) {
1801                 MRVL_LOG(ERR, "Flowctrl parameter is not supported");
1802
1803                 return -EINVAL;
1804         }
1805
1806         if (fc_conf->mode == RTE_FC_NONE ||
1807             fc_conf->mode == RTE_FC_RX_PAUSE) {
1808                 int ret, en;
1809
1810                 en = fc_conf->mode == RTE_FC_NONE ? 0 : 1;
1811                 ret = pp2_ppio_set_rx_pause(priv->ppio, en);
1812                 if (ret)
1813                         MRVL_LOG(ERR,
1814                                 "Failed to change flowctrl on RX side");
1815
1816                 return ret;
1817         }
1818
1819         return 0;
1820 }
1821
1822 /**
1823  * Update RSS hash configuration
1824  *
1825  * @param dev
1826  *   Pointer to Ethernet device structure.
1827  * @param rss_conf
1828  *   Pointer to RSS configuration.
1829  *
1830  * @return
1831  *   0 on success, negative error value otherwise.
1832  */
1833 static int
1834 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1835                      struct rte_eth_rss_conf *rss_conf)
1836 {
1837         struct mrvl_priv *priv = dev->data->dev_private;
1838
1839         if (priv->isolated)
1840                 return -ENOTSUP;
1841
1842         return mrvl_configure_rss(priv, rss_conf);
1843 }
1844
1845 /**
1846  * DPDK callback to get RSS hash configuration.
1847  *
1848  * @param dev
1849  *   Pointer to Ethernet device structure.
1850  * @rss_conf
1851  *   Pointer to RSS configuration.
1852  *
1853  * @return
1854  *   Always 0.
1855  */
1856 static int
1857 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1858                        struct rte_eth_rss_conf *rss_conf)
1859 {
1860         struct mrvl_priv *priv = dev->data->dev_private;
1861         enum pp2_ppio_hash_type hash_type =
1862                 priv->ppio_params.inqs_params.hash_type;
1863
1864         rss_conf->rss_key = NULL;
1865
1866         if (hash_type == PP2_PPIO_HASH_T_NONE)
1867                 rss_conf->rss_hf = 0;
1868         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1869                 rss_conf->rss_hf = ETH_RSS_IPV4;
1870         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1871                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1872         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1873                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1874
1875         return 0;
1876 }
1877
1878 /**
1879  * DPDK callback to get rte_flow callbacks.
1880  *
1881  * @param dev
1882  *   Pointer to the device structure.
1883  * @param filer_type
1884  *   Flow filter type.
1885  * @param filter_op
1886  *   Flow filter operation.
1887  * @param arg
1888  *   Pointer to pass the flow ops.
1889  *
1890  * @return
1891  *   0 on success, negative error value otherwise.
1892  */
1893 static int
1894 mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
1895                      enum rte_filter_type filter_type,
1896                      enum rte_filter_op filter_op, void *arg)
1897 {
1898         switch (filter_type) {
1899         case RTE_ETH_FILTER_GENERIC:
1900                 if (filter_op != RTE_ETH_FILTER_GET)
1901                         return -EINVAL;
1902                 *(const void **)arg = &mrvl_flow_ops;
1903                 return 0;
1904         default:
1905                 MRVL_LOG(WARNING, "Filter type (%d) not supported",
1906                                 filter_type);
1907                 return -EINVAL;
1908         }
1909 }
1910
1911 /**
1912  * DPDK callback to get rte_mtr callbacks.
1913  *
1914  * @param dev
1915  *   Pointer to the device structure.
1916  * @param ops
1917  *   Pointer to pass the mtr ops.
1918  *
1919  * @return
1920  *   Always 0.
1921  */
1922 static int
1923 mrvl_mtr_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
1924 {
1925         *(const void **)ops = &mrvl_mtr_ops;
1926
1927         return 0;
1928 }
1929
1930 /**
1931  * DPDK callback to get rte_tm callbacks.
1932  *
1933  * @param dev
1934  *   Pointer to the device structure.
1935  * @param ops
1936  *   Pointer to pass the tm ops.
1937  *
1938  * @return
1939  *   Always 0.
1940  */
1941 static int
1942 mrvl_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
1943 {
1944         *(const void **)ops = &mrvl_tm_ops;
1945
1946         return 0;
1947 }
1948
1949 static const struct eth_dev_ops mrvl_ops = {
1950         .dev_configure = mrvl_dev_configure,
1951         .dev_start = mrvl_dev_start,
1952         .dev_stop = mrvl_dev_stop,
1953         .dev_set_link_up = mrvl_dev_set_link_up,
1954         .dev_set_link_down = mrvl_dev_set_link_down,
1955         .dev_close = mrvl_dev_close,
1956         .link_update = mrvl_link_update,
1957         .promiscuous_enable = mrvl_promiscuous_enable,
1958         .allmulticast_enable = mrvl_allmulticast_enable,
1959         .promiscuous_disable = mrvl_promiscuous_disable,
1960         .allmulticast_disable = mrvl_allmulticast_disable,
1961         .mac_addr_remove = mrvl_mac_addr_remove,
1962         .mac_addr_add = mrvl_mac_addr_add,
1963         .mac_addr_set = mrvl_mac_addr_set,
1964         .mtu_set = mrvl_mtu_set,
1965         .stats_get = mrvl_stats_get,
1966         .stats_reset = mrvl_stats_reset,
1967         .xstats_get = mrvl_xstats_get,
1968         .xstats_reset = mrvl_xstats_reset,
1969         .xstats_get_names = mrvl_xstats_get_names,
1970         .dev_infos_get = mrvl_dev_infos_get,
1971         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
1972         .rxq_info_get = mrvl_rxq_info_get,
1973         .txq_info_get = mrvl_txq_info_get,
1974         .vlan_filter_set = mrvl_vlan_filter_set,
1975         .tx_queue_start = mrvl_tx_queue_start,
1976         .tx_queue_stop = mrvl_tx_queue_stop,
1977         .rx_queue_setup = mrvl_rx_queue_setup,
1978         .rx_queue_release = mrvl_rx_queue_release,
1979         .tx_queue_setup = mrvl_tx_queue_setup,
1980         .tx_queue_release = mrvl_tx_queue_release,
1981         .flow_ctrl_get = mrvl_flow_ctrl_get,
1982         .flow_ctrl_set = mrvl_flow_ctrl_set,
1983         .rss_hash_update = mrvl_rss_hash_update,
1984         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1985         .filter_ctrl = mrvl_eth_filter_ctrl,
1986         .mtr_ops_get = mrvl_mtr_ops_get,
1987         .tm_ops_get = mrvl_tm_ops_get,
1988 };
1989
1990 /**
1991  * Return packet type information and l3/l4 offsets.
1992  *
1993  * @param desc
1994  *   Pointer to the received packet descriptor.
1995  * @param l3_offset
1996  *   l3 packet offset.
1997  * @param l4_offset
1998  *   l4 packet offset.
1999  *
2000  * @return
2001  *   Packet type information.
2002  */
2003 static inline uint64_t
2004 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
2005                                     uint8_t *l3_offset, uint8_t *l4_offset)
2006 {
2007         enum pp2_inq_l3_type l3_type;
2008         enum pp2_inq_l4_type l4_type;
2009         enum pp2_inq_vlan_tag vlan_tag;
2010         uint64_t packet_type;
2011
2012         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
2013         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
2014         pp2_ppio_inq_desc_get_vlan_tag(desc, &vlan_tag);
2015
2016         packet_type = RTE_PTYPE_L2_ETHER;
2017
2018         switch (vlan_tag) {
2019         case PP2_INQ_VLAN_TAG_SINGLE:
2020                 packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
2021                 break;
2022         case PP2_INQ_VLAN_TAG_DOUBLE:
2023         case PP2_INQ_VLAN_TAG_TRIPLE:
2024                 packet_type |= RTE_PTYPE_L2_ETHER_QINQ;
2025                 break;
2026         default:
2027                 break;
2028         }
2029
2030         switch (l3_type) {
2031         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
2032                 packet_type |= RTE_PTYPE_L3_IPV4;
2033                 break;
2034         case PP2_INQ_L3_TYPE_IPV4_OK:
2035                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
2036                 break;
2037         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
2038                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
2039                 break;
2040         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
2041                 packet_type |= RTE_PTYPE_L3_IPV6;
2042                 break;
2043         case PP2_INQ_L3_TYPE_IPV6_EXT:
2044                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
2045                 break;
2046         case PP2_INQ_L3_TYPE_ARP:
2047                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
2048                 /*
2049                  * In case of ARP l4_offset is set to wrong value.
2050                  * Set it to proper one so that later on mbuf->l3_len can be
2051                  * calculated subtracting l4_offset and l3_offset.
2052                  */
2053                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
2054                 break;
2055         default:
2056                 MRVL_LOG(DEBUG, "Failed to recognise l3 packet type");
2057                 break;
2058         }
2059
2060         switch (l4_type) {
2061         case PP2_INQ_L4_TYPE_TCP:
2062                 packet_type |= RTE_PTYPE_L4_TCP;
2063                 break;
2064         case PP2_INQ_L4_TYPE_UDP:
2065                 packet_type |= RTE_PTYPE_L4_UDP;
2066                 break;
2067         default:
2068                 MRVL_LOG(DEBUG, "Failed to recognise l4 packet type");
2069                 break;
2070         }
2071
2072         return packet_type;
2073 }
2074
2075 /**
2076  * Get offload information from the received packet descriptor.
2077  *
2078  * @param desc
2079  *   Pointer to the received packet descriptor.
2080  *
2081  * @return
2082  *   Mbuf offload flags.
2083  */
2084 static inline uint64_t
2085 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
2086 {
2087         uint64_t flags;
2088         enum pp2_inq_desc_status status;
2089
2090         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
2091         if (unlikely(status != PP2_DESC_ERR_OK))
2092                 flags = PKT_RX_IP_CKSUM_BAD;
2093         else
2094                 flags = PKT_RX_IP_CKSUM_GOOD;
2095
2096         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
2097         if (unlikely(status != PP2_DESC_ERR_OK))
2098                 flags |= PKT_RX_L4_CKSUM_BAD;
2099         else
2100                 flags |= PKT_RX_L4_CKSUM_GOOD;
2101
2102         return flags;
2103 }
2104
2105 /**
2106  * DPDK callback for receive.
2107  *
2108  * @param rxq
2109  *   Generic pointer to the receive queue.
2110  * @param rx_pkts
2111  *   Array to store received packets.
2112  * @param nb_pkts
2113  *   Maximum number of packets in array.
2114  *
2115  * @return
2116  *   Number of packets successfully received.
2117  */
2118 static uint16_t
2119 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2120 {
2121         struct mrvl_rxq *q = rxq;
2122         struct pp2_ppio_desc descs[nb_pkts];
2123         struct pp2_bpool *bpool;
2124         int i, ret, rx_done = 0;
2125         int num;
2126         struct pp2_hif *hif;
2127         unsigned int core_id = rte_lcore_id();
2128
2129         hif = mrvl_get_hif(q->priv, core_id);
2130
2131         if (unlikely(!q->priv->ppio || !hif))
2132                 return 0;
2133
2134         bpool = q->priv->bpool;
2135
2136         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
2137                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
2138         if (unlikely(ret < 0)) {
2139                 MRVL_LOG(ERR, "Failed to receive packets");
2140                 return 0;
2141         }
2142         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
2143
2144         for (i = 0; i < nb_pkts; i++) {
2145                 struct rte_mbuf *mbuf;
2146                 uint8_t l3_offset, l4_offset;
2147                 enum pp2_inq_desc_status status;
2148                 uint64_t addr;
2149
2150                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2151                         struct pp2_ppio_desc *pref_desc;
2152                         u64 pref_addr;
2153
2154                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
2155                         pref_addr = cookie_addr_high |
2156                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
2157                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
2158                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
2159                 }
2160
2161                 addr = cookie_addr_high |
2162                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
2163                 mbuf = (struct rte_mbuf *)addr;
2164                 rte_pktmbuf_reset(mbuf);
2165
2166                 /* drop packet in case of mac, overrun or resource error */
2167                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
2168                 if (unlikely(status != PP2_DESC_ERR_OK)) {
2169                         struct pp2_buff_inf binf = {
2170                                 .addr = rte_mbuf_data_iova_default(mbuf),
2171                                 .cookie = (uint64_t)mbuf,
2172                         };
2173
2174                         pp2_bpool_put_buff(hif, bpool, &binf);
2175                         mrvl_port_bpool_size
2176                                 [bpool->pp2_id][bpool->id][core_id]++;
2177                         q->drop_mac++;
2178                         continue;
2179                 }
2180
2181                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
2182                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
2183                 mbuf->data_len = mbuf->pkt_len;
2184                 mbuf->port = q->port_id;
2185                 mbuf->packet_type =
2186                         mrvl_desc_to_packet_type_and_offset(&descs[i],
2187                                                             &l3_offset,
2188                                                             &l4_offset);
2189                 mbuf->l2_len = l3_offset;
2190                 mbuf->l3_len = l4_offset - l3_offset;
2191
2192                 if (likely(q->cksum_enabled))
2193                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
2194
2195                 rx_pkts[rx_done++] = mbuf;
2196                 q->bytes_recv += mbuf->pkt_len;
2197         }
2198
2199         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
2200                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
2201
2202                 if (unlikely(num <= q->priv->bpool_min_size ||
2203                              (!rx_done && num < q->priv->bpool_init_size))) {
2204                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
2205                         if (ret)
2206                                 MRVL_LOG(ERR, "Failed to fill bpool");
2207                 } else if (unlikely(num > q->priv->bpool_max_size)) {
2208                         int i;
2209                         int pkt_to_remove = num - q->priv->bpool_init_size;
2210                         struct rte_mbuf *mbuf;
2211                         struct pp2_buff_inf buff;
2212
2213                         MRVL_LOG(DEBUG,
2214                                 "port-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)",
2215                                 bpool->pp2_id, q->priv->ppio->port_id,
2216                                 bpool->id, pkt_to_remove, num,
2217                                 q->priv->bpool_init_size);
2218
2219                         for (i = 0; i < pkt_to_remove; i++) {
2220                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
2221                                 if (ret)
2222                                         break;
2223                                 mbuf = (struct rte_mbuf *)
2224                                         (cookie_addr_high | buff.cookie);
2225                                 rte_pktmbuf_free(mbuf);
2226                         }
2227                         mrvl_port_bpool_size
2228                                 [bpool->pp2_id][bpool->id][core_id] -= i;
2229                 }
2230                 rte_spinlock_unlock(&q->priv->lock);
2231         }
2232
2233         return rx_done;
2234 }
2235
2236 /**
2237  * Prepare offload information.
2238  *
2239  * @param ol_flags
2240  *   Offload flags.
2241  * @param packet_type
2242  *   Packet type bitfield.
2243  * @param l3_type
2244  *   Pointer to the pp2_ouq_l3_type structure.
2245  * @param l4_type
2246  *   Pointer to the pp2_outq_l4_type structure.
2247  * @param gen_l3_cksum
2248  *   Will be set to 1 in case l3 checksum is computed.
2249  * @param l4_cksum
2250  *   Will be set to 1 in case l4 checksum is computed.
2251  *
2252  * @return
2253  *   0 on success, negative error value otherwise.
2254  */
2255 static inline int
2256 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
2257                         enum pp2_outq_l3_type *l3_type,
2258                         enum pp2_outq_l4_type *l4_type,
2259                         int *gen_l3_cksum,
2260                         int *gen_l4_cksum)
2261 {
2262         /*
2263          * Based on ol_flags prepare information
2264          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
2265          * for offloading.
2266          */
2267         if (ol_flags & PKT_TX_IPV4) {
2268                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
2269                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
2270         } else if (ol_flags & PKT_TX_IPV6) {
2271                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
2272                 /* no checksum for ipv6 header */
2273                 *gen_l3_cksum = 0;
2274         } else {
2275                 /* if something different then stop processing */
2276                 return -1;
2277         }
2278
2279         ol_flags &= PKT_TX_L4_MASK;
2280         if ((packet_type & RTE_PTYPE_L4_TCP) &&
2281             ol_flags == PKT_TX_TCP_CKSUM) {
2282                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
2283                 *gen_l4_cksum = 1;
2284         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
2285                    ol_flags == PKT_TX_UDP_CKSUM) {
2286                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
2287                 *gen_l4_cksum = 1;
2288         } else {
2289                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
2290                 /* no checksum for other type */
2291                 *gen_l4_cksum = 0;
2292         }
2293
2294         return 0;
2295 }
2296
2297 /**
2298  * Release already sent buffers to bpool (buffer-pool).
2299  *
2300  * @param ppio
2301  *   Pointer to the port structure.
2302  * @param hif
2303  *   Pointer to the MUSDK hardware interface.
2304  * @param sq
2305  *   Pointer to the shadow queue.
2306  * @param qid
2307  *   Queue id number.
2308  * @param force
2309  *   Force releasing packets.
2310  */
2311 static inline void
2312 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
2313                        unsigned int core_id, struct mrvl_shadow_txq *sq,
2314                        int qid, int force)
2315 {
2316         struct buff_release_entry *entry;
2317         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
2318         int i;
2319
2320         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
2321
2322         sq->num_to_release += nb_done;
2323
2324         if (likely(!force &&
2325                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
2326                 return;
2327
2328         nb_done = sq->num_to_release;
2329         sq->num_to_release = 0;
2330
2331         for (i = 0; i < nb_done; i++) {
2332                 entry = &sq->ent[sq->tail + num];
2333                 if (unlikely(!entry->buff.addr)) {
2334                         MRVL_LOG(ERR,
2335                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!",
2336                                 sq->tail, (u64)entry->buff.cookie,
2337                                 (u64)entry->buff.addr);
2338                         skip_bufs = 1;
2339                         goto skip;
2340                 }
2341
2342                 if (unlikely(!entry->bpool)) {
2343                         struct rte_mbuf *mbuf;
2344
2345                         mbuf = (struct rte_mbuf *)
2346                                (cookie_addr_high | entry->buff.cookie);
2347                         rte_pktmbuf_free(mbuf);
2348                         skip_bufs = 1;
2349                         goto skip;
2350                 }
2351
2352                 mrvl_port_bpool_size
2353                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
2354                 num++;
2355                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
2356                         goto skip;
2357                 continue;
2358 skip:
2359                 if (likely(num))
2360                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2361                 num += skip_bufs;
2362                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2363                 sq->size -= num;
2364                 num = 0;
2365                 skip_bufs = 0;
2366         }
2367
2368         if (likely(num)) {
2369                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2370                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2371                 sq->size -= num;
2372         }
2373 }
2374
2375 /**
2376  * DPDK callback for transmit.
2377  *
2378  * @param txq
2379  *   Generic pointer transmit queue.
2380  * @param tx_pkts
2381  *   Packets to transmit.
2382  * @param nb_pkts
2383  *   Number of packets in array.
2384  *
2385  * @return
2386  *   Number of packets successfully transmitted.
2387  */
2388 static uint16_t
2389 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2390 {
2391         struct mrvl_txq *q = txq;
2392         struct mrvl_shadow_txq *sq;
2393         struct pp2_hif *hif;
2394         struct pp2_ppio_desc descs[nb_pkts];
2395         unsigned int core_id = rte_lcore_id();
2396         int i, ret, bytes_sent = 0;
2397         uint16_t num, sq_free_size;
2398         uint64_t addr;
2399
2400         hif = mrvl_get_hif(q->priv, core_id);
2401         sq = &q->shadow_txqs[core_id];
2402
2403         if (unlikely(!q->priv->ppio || !hif))
2404                 return 0;
2405
2406         if (sq->size)
2407                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2408                                        sq, q->queue_id, 0);
2409
2410         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2411         if (unlikely(nb_pkts > sq_free_size)) {
2412                 MRVL_LOG(DEBUG,
2413                         "No room in shadow queue for %d packets! %d packets will be sent.",
2414                         nb_pkts, sq_free_size);
2415                 nb_pkts = sq_free_size;
2416         }
2417
2418         for (i = 0; i < nb_pkts; i++) {
2419                 struct rte_mbuf *mbuf = tx_pkts[i];
2420                 int gen_l3_cksum, gen_l4_cksum;
2421                 enum pp2_outq_l3_type l3_type;
2422                 enum pp2_outq_l4_type l4_type;
2423
2424                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2425                         struct rte_mbuf *pref_pkt_hdr;
2426
2427                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2428                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2429                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2430                 }
2431
2432                 sq->ent[sq->head].buff.cookie = (uint64_t)mbuf;
2433                 sq->ent[sq->head].buff.addr =
2434                         rte_mbuf_data_iova_default(mbuf);
2435                 sq->ent[sq->head].bpool =
2436                         (unlikely(mbuf->port >= RTE_MAX_ETHPORTS ||
2437                          mbuf->refcnt > 1)) ? NULL :
2438                          mrvl_port_to_bpool_lookup[mbuf->port];
2439                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
2440                 sq->size++;
2441
2442                 pp2_ppio_outq_desc_reset(&descs[i]);
2443                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
2444                                                  rte_pktmbuf_iova(mbuf));
2445                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
2446                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
2447                                                rte_pktmbuf_pkt_len(mbuf));
2448
2449                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2450                 /*
2451                  * in case unsupported ol_flags were passed
2452                  * do not update descriptor offload information
2453                  */
2454                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
2455                                               &l3_type, &l4_type, &gen_l3_cksum,
2456                                               &gen_l4_cksum);
2457                 if (unlikely(ret))
2458                         continue;
2459
2460                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2461                                                   mbuf->l2_len,
2462                                                   mbuf->l2_len + mbuf->l3_len,
2463                                                   gen_l3_cksum, gen_l4_cksum);
2464         }
2465
2466         num = nb_pkts;
2467         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2468         /* number of packets that were not sent */
2469         if (unlikely(num > nb_pkts)) {
2470                 for (i = nb_pkts; i < num; i++) {
2471                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2472                                 MRVL_PP2_TX_SHADOWQ_MASK;
2473                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
2474                         bytes_sent -=
2475                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2476                 }
2477                 sq->size -= num - nb_pkts;
2478         }
2479
2480         q->bytes_sent += bytes_sent;
2481
2482         return nb_pkts;
2483 }
2484
2485 /**
2486  * Initialize packet processor.
2487  *
2488  * @return
2489  *   0 on success, negative error value otherwise.
2490  */
2491 static int
2492 mrvl_init_pp2(void)
2493 {
2494         struct pp2_init_params init_params;
2495
2496         memset(&init_params, 0, sizeof(init_params));
2497         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
2498         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
2499         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
2500
2501         return pp2_init(&init_params);
2502 }
2503
2504 /**
2505  * Deinitialize packet processor.
2506  *
2507  * @return
2508  *   0 on success, negative error value otherwise.
2509  */
2510 static void
2511 mrvl_deinit_pp2(void)
2512 {
2513         pp2_deinit();
2514 }
2515
2516 /**
2517  * Create private device structure.
2518  *
2519  * @param dev_name
2520  *   Pointer to the port name passed in the initialization parameters.
2521  *
2522  * @return
2523  *   Pointer to the newly allocated private device structure.
2524  */
2525 static struct mrvl_priv *
2526 mrvl_priv_create(const char *dev_name)
2527 {
2528         struct pp2_bpool_params bpool_params;
2529         char match[MRVL_MATCH_LEN];
2530         struct mrvl_priv *priv;
2531         int ret, bpool_bit;
2532
2533         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
2534         if (!priv)
2535                 return NULL;
2536
2537         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
2538                                        &priv->pp_id, &priv->ppio_id);
2539         if (ret)
2540                 goto out_free_priv;
2541
2542         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
2543                                      PP2_BPOOL_NUM_POOLS);
2544         if (bpool_bit < 0)
2545                 goto out_free_priv;
2546         priv->bpool_bit = bpool_bit;
2547
2548         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
2549                  priv->bpool_bit);
2550         memset(&bpool_params, 0, sizeof(bpool_params));
2551         bpool_params.match = match;
2552         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
2553         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
2554         if (ret)
2555                 goto out_clear_bpool_bit;
2556
2557         priv->ppio_params.type = PP2_PPIO_T_NIC;
2558         rte_spinlock_init(&priv->lock);
2559
2560         return priv;
2561 out_clear_bpool_bit:
2562         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2563 out_free_priv:
2564         rte_free(priv);
2565         return NULL;
2566 }
2567
2568 /**
2569  * Create device representing Ethernet port.
2570  *
2571  * @param name
2572  *   Pointer to the port's name.
2573  *
2574  * @return
2575  *   0 on success, negative error value otherwise.
2576  */
2577 static int
2578 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
2579 {
2580         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
2581         struct rte_eth_dev *eth_dev;
2582         struct mrvl_priv *priv;
2583         struct ifreq req;
2584
2585         eth_dev = rte_eth_dev_allocate(name);
2586         if (!eth_dev)
2587                 return -ENOMEM;
2588
2589         priv = mrvl_priv_create(name);
2590         if (!priv) {
2591                 ret = -ENOMEM;
2592                 goto out_free_dev;
2593         }
2594
2595         eth_dev->data->mac_addrs =
2596                 rte_zmalloc("mac_addrs",
2597                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2598         if (!eth_dev->data->mac_addrs) {
2599                 MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
2600                 ret = -ENOMEM;
2601                 goto out_free_priv;
2602         }
2603
2604         memset(&req, 0, sizeof(req));
2605         strcpy(req.ifr_name, name);
2606         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2607         if (ret)
2608                 goto out_free_mac;
2609
2610         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2611                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2612
2613         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2614         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
2615         eth_dev->data->kdrv = RTE_KDRV_NONE;
2616         eth_dev->data->dev_private = priv;
2617         eth_dev->device = &vdev->device;
2618         eth_dev->dev_ops = &mrvl_ops;
2619
2620         rte_eth_dev_probing_finish(eth_dev);
2621         return 0;
2622 out_free_mac:
2623         rte_free(eth_dev->data->mac_addrs);
2624 out_free_dev:
2625         rte_eth_dev_release_port(eth_dev);
2626 out_free_priv:
2627         rte_free(priv);
2628
2629         return ret;
2630 }
2631
2632 /**
2633  * Cleanup previously created device representing Ethernet port.
2634  *
2635  * @param name
2636  *   Pointer to the port name.
2637  */
2638 static void
2639 mrvl_eth_dev_destroy(const char *name)
2640 {
2641         struct rte_eth_dev *eth_dev;
2642         struct mrvl_priv *priv;
2643
2644         eth_dev = rte_eth_dev_allocated(name);
2645         if (!eth_dev)
2646                 return;
2647
2648         priv = eth_dev->data->dev_private;
2649         pp2_bpool_deinit(priv->bpool);
2650         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2651         rte_free(priv);
2652         rte_free(eth_dev->data->mac_addrs);
2653         rte_eth_dev_release_port(eth_dev);
2654 }
2655
2656 /**
2657  * Callback used by rte_kvargs_process() during argument parsing.
2658  *
2659  * @param key
2660  *   Pointer to the parsed key (unused).
2661  * @param value
2662  *   Pointer to the parsed value.
2663  * @param extra_args
2664  *   Pointer to the extra arguments which contains address of the
2665  *   table of pointers to parsed interface names.
2666  *
2667  * @return
2668  *   Always 0.
2669  */
2670 static int
2671 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2672                  void *extra_args)
2673 {
2674         struct mrvl_ifnames *ifnames = extra_args;
2675
2676         ifnames->names[ifnames->idx++] = value;
2677
2678         return 0;
2679 }
2680
2681 /**
2682  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2683  */
2684 static void
2685 mrvl_deinit_hifs(void)
2686 {
2687         int i;
2688
2689         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
2690                 if (hifs[i])
2691                         pp2_hif_deinit(hifs[i]);
2692         }
2693         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
2694         memset(hifs, 0, sizeof(hifs));
2695 }
2696
2697 /**
2698  * DPDK callback to register the virtual device.
2699  *
2700  * @param vdev
2701  *   Pointer to the virtual device.
2702  *
2703  * @return
2704  *   0 on success, negative error value otherwise.
2705  */
2706 static int
2707 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2708 {
2709         struct rte_kvargs *kvlist;
2710         struct mrvl_ifnames ifnames;
2711         int ret = -EINVAL;
2712         uint32_t i, ifnum, cfgnum;
2713         const char *params;
2714
2715         params = rte_vdev_device_args(vdev);
2716         if (!params)
2717                 return -EINVAL;
2718
2719         kvlist = rte_kvargs_parse(params, valid_args);
2720         if (!kvlist)
2721                 return -EINVAL;
2722
2723         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2724         if (ifnum > RTE_DIM(ifnames.names))
2725                 goto out_free_kvlist;
2726
2727         ifnames.idx = 0;
2728         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2729                            mrvl_get_ifnames, &ifnames);
2730
2731
2732         /*
2733          * The below system initialization should be done only once,
2734          * on the first provided configuration file
2735          */
2736         if (!mrvl_qos_cfg) {
2737                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2738                 MRVL_LOG(INFO, "Parsing config file!");
2739                 if (cfgnum > 1) {
2740                         MRVL_LOG(ERR, "Cannot handle more than one config file!");
2741                         goto out_free_kvlist;
2742                 } else if (cfgnum == 1) {
2743                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2744                                            mrvl_get_qoscfg, &mrvl_qos_cfg);
2745                 }
2746         }
2747
2748         if (mrvl_dev_num)
2749                 goto init_devices;
2750
2751         MRVL_LOG(INFO, "Perform MUSDK initializations");
2752
2753         ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist);
2754         if (ret)
2755                 goto out_free_kvlist;
2756
2757         ret = mrvl_init_pp2();
2758         if (ret) {
2759                 MRVL_LOG(ERR, "Failed to init PP!");
2760                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2761                 goto out_free_kvlist;
2762         }
2763
2764         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2765         memset(mrvl_port_to_bpool_lookup, 0, sizeof(mrvl_port_to_bpool_lookup));
2766
2767         mrvl_lcore_first = RTE_MAX_LCORE;
2768         mrvl_lcore_last = 0;
2769
2770 init_devices:
2771         for (i = 0; i < ifnum; i++) {
2772                 MRVL_LOG(INFO, "Creating %s", ifnames.names[i]);
2773                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
2774                 if (ret)
2775                         goto out_cleanup;
2776         }
2777         mrvl_dev_num += ifnum;
2778
2779         rte_kvargs_free(kvlist);
2780
2781         return 0;
2782 out_cleanup:
2783         for (; i > 0; i--)
2784                 mrvl_eth_dev_destroy(ifnames.names[i]);
2785
2786         if (mrvl_dev_num == 0) {
2787                 mrvl_deinit_pp2();
2788                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2789         }
2790 out_free_kvlist:
2791         rte_kvargs_free(kvlist);
2792
2793         return ret;
2794 }
2795
2796 /**
2797  * DPDK callback to remove virtual device.
2798  *
2799  * @param vdev
2800  *   Pointer to the removed virtual device.
2801  *
2802  * @return
2803  *   0 on success, negative error value otherwise.
2804  */
2805 static int
2806 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2807 {
2808         int i;
2809         const char *name;
2810
2811         name = rte_vdev_device_name(vdev);
2812         if (!name)
2813                 return -EINVAL;
2814
2815         MRVL_LOG(INFO, "Removing %s", name);
2816
2817         RTE_ETH_FOREACH_DEV(i) { /* FIXME: removing all devices! */
2818                 char ifname[RTE_ETH_NAME_MAX_LEN];
2819
2820                 rte_eth_dev_get_name_by_port(i, ifname);
2821                 mrvl_eth_dev_destroy(ifname);
2822                 mrvl_dev_num--;
2823         }
2824
2825         if (mrvl_dev_num == 0) {
2826                 MRVL_LOG(INFO, "Perform MUSDK deinit");
2827                 mrvl_deinit_hifs();
2828                 mrvl_deinit_pp2();
2829                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2830         }
2831
2832         return 0;
2833 }
2834
2835 static struct rte_vdev_driver pmd_mrvl_drv = {
2836         .probe = rte_pmd_mrvl_probe,
2837         .remove = rte_pmd_mrvl_remove,
2838 };
2839
2840 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv);
2841 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2);
2842
2843 RTE_INIT(mrvl_init_log)
2844 {
2845         mrvl_logtype = rte_log_register("pmd.net.mvpp2");
2846         if (mrvl_logtype >= 0)
2847                 rte_log_set_level(mrvl_logtype, RTE_LOG_NOTICE);
2848 }