net/mvpp2: make private variables static
[dpdk.git] / drivers / net / mvpp2 / mrvl_ethdev.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017 Marvell International Ltd.
3  * Copyright(c) 2017 Semihalf.
4  * All rights reserved.
5  */
6
7 #include <rte_ethdev_driver.h>
8 #include <rte_kvargs.h>
9 #include <rte_log.h>
10 #include <rte_malloc.h>
11 #include <rte_bus_vdev.h>
12
13 /* Unluckily, container_of is defined by both DPDK and MUSDK,
14  * we'll declare only one version.
15  *
16  * Note that it is not used in this PMD anyway.
17  */
18 #ifdef container_of
19 #undef container_of
20 #endif
21
22 #include <fcntl.h>
23 #include <linux/ethtool.h>
24 #include <linux/sockios.h>
25 #include <net/if.h>
26 #include <net/if_arp.h>
27 #include <sys/ioctl.h>
28 #include <sys/socket.h>
29 #include <sys/stat.h>
30 #include <sys/types.h>
31
32 #include <rte_mvep_common.h>
33 #include "mrvl_ethdev.h"
34 #include "mrvl_qos.h"
35
36 /* bitmask with reserved hifs */
37 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
38 /* bitmask with reserved bpools */
39 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
40 /* bitmask with reserved kernel RSS tables */
41 #define MRVL_MUSDK_RSS_RESERVED 0x01
42 /* maximum number of available hifs */
43 #define MRVL_MUSDK_HIFS_MAX 9
44
45 /* prefetch shift */
46 #define MRVL_MUSDK_PREFETCH_SHIFT 2
47
48 /* TCAM has 25 entries reserved for uc/mc filter entries */
49 #define MRVL_MAC_ADDRS_MAX 25
50 #define MRVL_MATCH_LEN 16
51 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
52 /* Maximum allowable packet size */
53 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
54
55 #define MRVL_IFACE_NAME_ARG "iface"
56 #define MRVL_CFG_ARG "cfg"
57
58 #define MRVL_BURST_SIZE 64
59
60 #define MRVL_ARP_LENGTH 28
61
62 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
63
64 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
65 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
66
67 /** Port Rx offload capabilities */
68 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
69                           DEV_RX_OFFLOAD_JUMBO_FRAME | \
70                           DEV_RX_OFFLOAD_CRC_STRIP | \
71                           DEV_RX_OFFLOAD_CHECKSUM)
72
73 /** Port Tx offloads capabilities */
74 #define MRVL_TX_OFFLOADS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
75                           DEV_TX_OFFLOAD_UDP_CKSUM | \
76                           DEV_TX_OFFLOAD_TCP_CKSUM)
77
78 static const char * const valid_args[] = {
79         MRVL_IFACE_NAME_ARG,
80         MRVL_CFG_ARG,
81         NULL
82 };
83
84 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
85 static struct pp2_hif *hifs[RTE_MAX_LCORE];
86 static int used_bpools[PP2_NUM_PKT_PROC] = {
87         [0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED
88 };
89
90 static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
91 static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
92 static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
93
94 int mrvl_logtype;
95
96 struct mrvl_ifnames {
97         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
98         int idx;
99 };
100
101 /*
102  * To use buffer harvesting based on loopback port shadow queue structure
103  * was introduced for buffers information bookkeeping.
104  *
105  * Before sending the packet, related buffer information (pp2_buff_inf) is
106  * stored in shadow queue. After packet is transmitted no longer used
107  * packet buffer is released back to it's original hardware pool,
108  * on condition it originated from interface.
109  * In case it  was generated by application itself i.e: mbuf->port field is
110  * 0xff then its released to software mempool.
111  */
112 struct mrvl_shadow_txq {
113         int head;           /* write index - used when sending buffers */
114         int tail;           /* read index - used when releasing buffers */
115         u16 size;           /* queue occupied size */
116         u16 num_to_release; /* number of buffers sent, that can be released */
117         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
118 };
119
120 struct mrvl_rxq {
121         struct mrvl_priv *priv;
122         struct rte_mempool *mp;
123         int queue_id;
124         int port_id;
125         int cksum_enabled;
126         uint64_t bytes_recv;
127         uint64_t drop_mac;
128 };
129
130 struct mrvl_txq {
131         struct mrvl_priv *priv;
132         int queue_id;
133         int port_id;
134         uint64_t bytes_sent;
135         struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE];
136         int tx_deferred_start;
137 };
138
139 static int mrvl_lcore_first;
140 static int mrvl_lcore_last;
141 static int mrvl_dev_num;
142
143 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
144 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
145                         struct pp2_hif *hif, unsigned int core_id,
146                         struct mrvl_shadow_txq *sq, int qid, int force);
147
148 #define MRVL_XSTATS_TBL_ENTRY(name) { \
149         #name, offsetof(struct pp2_ppio_statistics, name),      \
150         sizeof(((struct pp2_ppio_statistics *)0)->name)         \
151 }
152
153 /* Table with xstats data */
154 static struct {
155         const char *name;
156         unsigned int offset;
157         unsigned int size;
158 } mrvl_xstats_tbl[] = {
159         MRVL_XSTATS_TBL_ENTRY(rx_bytes),
160         MRVL_XSTATS_TBL_ENTRY(rx_packets),
161         MRVL_XSTATS_TBL_ENTRY(rx_unicast_packets),
162         MRVL_XSTATS_TBL_ENTRY(rx_errors),
163         MRVL_XSTATS_TBL_ENTRY(rx_fullq_dropped),
164         MRVL_XSTATS_TBL_ENTRY(rx_bm_dropped),
165         MRVL_XSTATS_TBL_ENTRY(rx_early_dropped),
166         MRVL_XSTATS_TBL_ENTRY(rx_fifo_dropped),
167         MRVL_XSTATS_TBL_ENTRY(rx_cls_dropped),
168         MRVL_XSTATS_TBL_ENTRY(tx_bytes),
169         MRVL_XSTATS_TBL_ENTRY(tx_packets),
170         MRVL_XSTATS_TBL_ENTRY(tx_unicast_packets),
171         MRVL_XSTATS_TBL_ENTRY(tx_errors)
172 };
173
174 static inline int
175 mrvl_get_bpool_size(int pp2_id, int pool_id)
176 {
177         int i;
178         int size = 0;
179
180         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
181                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
182
183         return size;
184 }
185
186 static inline int
187 mrvl_reserve_bit(int *bitmap, int max)
188 {
189         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
190
191         if (n >= max)
192                 return -1;
193
194         *bitmap |= 1 << n;
195
196         return n;
197 }
198
199 static int
200 mrvl_init_hif(int core_id)
201 {
202         struct pp2_hif_params params;
203         char match[MRVL_MATCH_LEN];
204         int ret;
205
206         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
207         if (ret < 0) {
208                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
209                 return ret;
210         }
211
212         snprintf(match, sizeof(match), "hif-%d", ret);
213         memset(&params, 0, sizeof(params));
214         params.match = match;
215         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
216         ret = pp2_hif_init(&params, &hifs[core_id]);
217         if (ret) {
218                 MRVL_LOG(ERR, "Failed to initialize hif %d", core_id);
219                 return ret;
220         }
221
222         return 0;
223 }
224
225 static inline struct pp2_hif*
226 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
227 {
228         int ret;
229
230         if (likely(hifs[core_id] != NULL))
231                 return hifs[core_id];
232
233         rte_spinlock_lock(&priv->lock);
234
235         ret = mrvl_init_hif(core_id);
236         if (ret < 0) {
237                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
238                 goto out;
239         }
240
241         if (core_id < mrvl_lcore_first)
242                 mrvl_lcore_first = core_id;
243
244         if (core_id > mrvl_lcore_last)
245                 mrvl_lcore_last = core_id;
246 out:
247         rte_spinlock_unlock(&priv->lock);
248
249         return hifs[core_id];
250 }
251
252 /**
253  * Configure rss based on dpdk rss configuration.
254  *
255  * @param priv
256  *   Pointer to private structure.
257  * @param rss_conf
258  *   Pointer to RSS configuration.
259  *
260  * @return
261  *   0 on success, negative error value otherwise.
262  */
263 static int
264 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
265 {
266         if (rss_conf->rss_key)
267                 MRVL_LOG(WARNING, "Changing hash key is not supported");
268
269         if (rss_conf->rss_hf == 0) {
270                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
271         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
272                 priv->ppio_params.inqs_params.hash_type =
273                         PP2_PPIO_HASH_T_2_TUPLE;
274         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
275                 priv->ppio_params.inqs_params.hash_type =
276                         PP2_PPIO_HASH_T_5_TUPLE;
277                 priv->rss_hf_tcp = 1;
278         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
279                 priv->ppio_params.inqs_params.hash_type =
280                         PP2_PPIO_HASH_T_5_TUPLE;
281                 priv->rss_hf_tcp = 0;
282         } else {
283                 return -EINVAL;
284         }
285
286         return 0;
287 }
288
289 /**
290  * Ethernet device configuration.
291  *
292  * Prepare the driver for a given number of TX and RX queues and
293  * configure RSS.
294  *
295  * @param dev
296  *   Pointer to Ethernet device structure.
297  *
298  * @return
299  *   0 on success, negative error value otherwise.
300  */
301 static int
302 mrvl_dev_configure(struct rte_eth_dev *dev)
303 {
304         struct mrvl_priv *priv = dev->data->dev_private;
305         int ret;
306
307         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
308             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
309                 MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
310                         dev->data->dev_conf.rxmode.mq_mode);
311                 return -EINVAL;
312         }
313
314         /* KEEP_CRC offload flag is not supported by PMD
315          * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
316          */
317         if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads)) {
318                 MRVL_LOG(INFO, "L2 CRC stripping is always enabled in hw");
319                 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
320         }
321
322         if (dev->data->dev_conf.rxmode.split_hdr_size) {
323                 MRVL_LOG(INFO, "Split headers not supported");
324                 return -EINVAL;
325         }
326
327         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
328                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
329                                  ETHER_HDR_LEN - ETHER_CRC_LEN;
330
331         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
332                                   dev->data->nb_rx_queues);
333         if (ret < 0)
334                 return ret;
335
336         ret = mrvl_configure_txqs(priv, dev->data->port_id,
337                                   dev->data->nb_tx_queues);
338         if (ret < 0)
339                 return ret;
340
341         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
342         priv->ppio_params.maintain_stats = 1;
343         priv->nb_rx_queues = dev->data->nb_rx_queues;
344
345         if (dev->data->nb_rx_queues == 1 &&
346             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
347                 MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
348                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
349
350                 return 0;
351         }
352
353         return mrvl_configure_rss(priv,
354                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
355 }
356
357 /**
358  * DPDK callback to change the MTU.
359  *
360  * Setting the MTU affects hardware MRU (packets larger than the MRU
361  * will be dropped).
362  *
363  * @param dev
364  *   Pointer to Ethernet device structure.
365  * @param mtu
366  *   New MTU.
367  *
368  * @return
369  *   0 on success, negative error value otherwise.
370  */
371 static int
372 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
373 {
374         struct mrvl_priv *priv = dev->data->dev_private;
375         /* extra MV_MH_SIZE bytes are required for Marvell tag */
376         uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
377         int ret;
378
379         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
380                 return -EINVAL;
381
382         if (!priv->ppio)
383                 return 0;
384
385         ret = pp2_ppio_set_mru(priv->ppio, mru);
386         if (ret)
387                 return ret;
388
389         return pp2_ppio_set_mtu(priv->ppio, mtu);
390 }
391
392 /**
393  * DPDK callback to bring the link up.
394  *
395  * @param dev
396  *   Pointer to Ethernet device structure.
397  *
398  * @return
399  *   0 on success, negative error value otherwise.
400  */
401 static int
402 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
403 {
404         struct mrvl_priv *priv = dev->data->dev_private;
405         int ret;
406
407         if (!priv->ppio)
408                 return -EPERM;
409
410         ret = pp2_ppio_enable(priv->ppio);
411         if (ret)
412                 return ret;
413
414         /*
415          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
416          * as pp2_ppio_enable() changes port->t_mode from default 0 to
417          * PP2_TRAFFIC_INGRESS_EGRESS.
418          *
419          * Set mtu to default DPDK value here.
420          */
421         ret = mrvl_mtu_set(dev, dev->data->mtu);
422         if (ret)
423                 pp2_ppio_disable(priv->ppio);
424
425         return ret;
426 }
427
428 /**
429  * DPDK callback to bring the link down.
430  *
431  * @param dev
432  *   Pointer to Ethernet device structure.
433  *
434  * @return
435  *   0 on success, negative error value otherwise.
436  */
437 static int
438 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
439 {
440         struct mrvl_priv *priv = dev->data->dev_private;
441
442         if (!priv->ppio)
443                 return -EPERM;
444
445         return pp2_ppio_disable(priv->ppio);
446 }
447
448 /**
449  * DPDK callback to start tx queue.
450  *
451  * @param dev
452  *   Pointer to Ethernet device structure.
453  * @param queue_id
454  *   Transmit queue index.
455  *
456  * @return
457  *   0 on success, negative error value otherwise.
458  */
459 static int
460 mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
461 {
462         struct mrvl_priv *priv = dev->data->dev_private;
463         int ret;
464
465         if (!priv)
466                 return -EPERM;
467
468         /* passing 1 enables given tx queue */
469         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1);
470         if (ret) {
471                 MRVL_LOG(ERR, "Failed to start txq %d", queue_id);
472                 return ret;
473         }
474
475         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
476
477         return 0;
478 }
479
480 /**
481  * DPDK callback to stop tx queue.
482  *
483  * @param dev
484  *   Pointer to Ethernet device structure.
485  * @param queue_id
486  *   Transmit queue index.
487  *
488  * @return
489  *   0 on success, negative error value otherwise.
490  */
491 static int
492 mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
493 {
494         struct mrvl_priv *priv = dev->data->dev_private;
495         int ret;
496
497         if (!priv->ppio)
498                 return -EPERM;
499
500         /* passing 0 disables given tx queue */
501         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0);
502         if (ret) {
503                 MRVL_LOG(ERR, "Failed to stop txq %d", queue_id);
504                 return ret;
505         }
506
507         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
508
509         return 0;
510 }
511
512 /**
513  * DPDK callback to start the device.
514  *
515  * @param dev
516  *   Pointer to Ethernet device structure.
517  *
518  * @return
519  *   0 on success, negative errno value on failure.
520  */
521 static int
522 mrvl_dev_start(struct rte_eth_dev *dev)
523 {
524         struct mrvl_priv *priv = dev->data->dev_private;
525         char match[MRVL_MATCH_LEN];
526         int ret = 0, i, def_init_size;
527
528         snprintf(match, sizeof(match), "ppio-%d:%d",
529                  priv->pp_id, priv->ppio_id);
530         priv->ppio_params.match = match;
531
532         /*
533          * Calculate the minimum bpool size for refill feature as follows:
534          * 2 default burst sizes multiply by number of rx queues.
535          * If the bpool size will be below this value, new buffers will
536          * be added to the pool.
537          */
538         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
539
540         /* In case initial bpool size configured in queues setup is
541          * smaller than minimum size add more buffers
542          */
543         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
544         if (priv->bpool_init_size < def_init_size) {
545                 int buffs_to_add = def_init_size - priv->bpool_init_size;
546
547                 priv->bpool_init_size += buffs_to_add;
548                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
549                 if (ret)
550                         MRVL_LOG(ERR, "Failed to add buffers to bpool");
551         }
552
553         /*
554          * Calculate the maximum bpool size for refill feature as follows:
555          * maximum number of descriptors in rx queue multiply by number
556          * of rx queues plus minimum bpool size.
557          * In case the bpool size will exceed this value, superfluous buffers
558          * will be removed
559          */
560         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
561                                 priv->bpool_min_size;
562
563         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
564         if (ret) {
565                 MRVL_LOG(ERR, "Failed to init ppio");
566                 return ret;
567         }
568
569         /*
570          * In case there are some some stale uc/mc mac addresses flush them
571          * here. It cannot be done during mrvl_dev_close() as port information
572          * is already gone at that point (due to pp2_ppio_deinit() in
573          * mrvl_dev_stop()).
574          */
575         if (!priv->uc_mc_flushed) {
576                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
577                 if (ret) {
578                         MRVL_LOG(ERR,
579                                 "Failed to flush uc/mc filter list");
580                         goto out;
581                 }
582                 priv->uc_mc_flushed = 1;
583         }
584
585         if (!priv->vlan_flushed) {
586                 ret = pp2_ppio_flush_vlan(priv->ppio);
587                 if (ret) {
588                         MRVL_LOG(ERR, "Failed to flush vlan list");
589                         /*
590                          * TODO
591                          * once pp2_ppio_flush_vlan() is supported jump to out
592                          * goto out;
593                          */
594                 }
595                 priv->vlan_flushed = 1;
596         }
597
598         /* For default QoS config, don't start classifier. */
599         if (mrvl_qos_cfg) {
600                 ret = mrvl_start_qos_mapping(priv);
601                 if (ret) {
602                         MRVL_LOG(ERR, "Failed to setup QoS mapping");
603                         goto out;
604                 }
605         }
606
607         ret = mrvl_dev_set_link_up(dev);
608         if (ret) {
609                 MRVL_LOG(ERR, "Failed to set link up");
610                 goto out;
611         }
612
613         /* start tx queues */
614         for (i = 0; i < dev->data->nb_tx_queues; i++) {
615                 struct mrvl_txq *txq = dev->data->tx_queues[i];
616
617                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
618
619                 if (!txq->tx_deferred_start)
620                         continue;
621
622                 /*
623                  * All txqs are started by default. Stop them
624                  * so that tx_deferred_start works as expected.
625                  */
626                 ret = mrvl_tx_queue_stop(dev, i);
627                 if (ret)
628                         goto out;
629         }
630
631         return 0;
632 out:
633         MRVL_LOG(ERR, "Failed to start device");
634         pp2_ppio_deinit(priv->ppio);
635         return ret;
636 }
637
638 /**
639  * Flush receive queues.
640  *
641  * @param dev
642  *   Pointer to Ethernet device structure.
643  */
644 static void
645 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
646 {
647         int i;
648
649         MRVL_LOG(INFO, "Flushing rx queues");
650         for (i = 0; i < dev->data->nb_rx_queues; i++) {
651                 int ret, num;
652
653                 do {
654                         struct mrvl_rxq *q = dev->data->rx_queues[i];
655                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
656
657                         num = MRVL_PP2_RXD_MAX;
658                         ret = pp2_ppio_recv(q->priv->ppio,
659                                             q->priv->rxq_map[q->queue_id].tc,
660                                             q->priv->rxq_map[q->queue_id].inq,
661                                             descs, (uint16_t *)&num);
662                 } while (ret == 0 && num);
663         }
664 }
665
666 /**
667  * Flush transmit shadow queues.
668  *
669  * @param dev
670  *   Pointer to Ethernet device structure.
671  */
672 static void
673 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
674 {
675         int i, j;
676         struct mrvl_txq *txq;
677
678         MRVL_LOG(INFO, "Flushing tx shadow queues");
679         for (i = 0; i < dev->data->nb_tx_queues; i++) {
680                 txq = (struct mrvl_txq *)dev->data->tx_queues[i];
681
682                 for (j = 0; j < RTE_MAX_LCORE; j++) {
683                         struct mrvl_shadow_txq *sq;
684
685                         if (!hifs[j])
686                                 continue;
687
688                         sq = &txq->shadow_txqs[j];
689                         mrvl_free_sent_buffers(txq->priv->ppio,
690                                 hifs[j], j, sq, txq->queue_id, 1);
691                         while (sq->tail != sq->head) {
692                                 uint64_t addr = cookie_addr_high |
693                                         sq->ent[sq->tail].buff.cookie;
694                                 rte_pktmbuf_free(
695                                         (struct rte_mbuf *)addr);
696                                 sq->tail = (sq->tail + 1) &
697                                             MRVL_PP2_TX_SHADOWQ_MASK;
698                         }
699                         memset(sq, 0, sizeof(*sq));
700                 }
701         }
702 }
703
704 /**
705  * Flush hardware bpool (buffer-pool).
706  *
707  * @param dev
708  *   Pointer to Ethernet device structure.
709  */
710 static void
711 mrvl_flush_bpool(struct rte_eth_dev *dev)
712 {
713         struct mrvl_priv *priv = dev->data->dev_private;
714         struct pp2_hif *hif;
715         uint32_t num;
716         int ret;
717         unsigned int core_id = rte_lcore_id();
718
719         if (core_id == LCORE_ID_ANY)
720                 core_id = 0;
721
722         hif = mrvl_get_hif(priv, core_id);
723
724         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
725         if (ret) {
726                 MRVL_LOG(ERR, "Failed to get bpool buffers number");
727                 return;
728         }
729
730         while (num--) {
731                 struct pp2_buff_inf inf;
732                 uint64_t addr;
733
734                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
735                 if (ret)
736                         break;
737
738                 addr = cookie_addr_high | inf.cookie;
739                 rte_pktmbuf_free((struct rte_mbuf *)addr);
740         }
741 }
742
743 /**
744  * DPDK callback to stop the device.
745  *
746  * @param dev
747  *   Pointer to Ethernet device structure.
748  */
749 static void
750 mrvl_dev_stop(struct rte_eth_dev *dev)
751 {
752         struct mrvl_priv *priv = dev->data->dev_private;
753
754         mrvl_dev_set_link_down(dev);
755         mrvl_flush_rx_queues(dev);
756         mrvl_flush_tx_shadow_queues(dev);
757         if (priv->cls_tbl) {
758                 pp2_cls_tbl_deinit(priv->cls_tbl);
759                 priv->cls_tbl = NULL;
760         }
761         if (priv->qos_tbl) {
762                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
763                 priv->qos_tbl = NULL;
764         }
765         if (priv->ppio)
766                 pp2_ppio_deinit(priv->ppio);
767         priv->ppio = NULL;
768
769         /* policer must be released after ppio deinitialization */
770         if (priv->policer) {
771                 pp2_cls_plcr_deinit(priv->policer);
772                 priv->policer = NULL;
773         }
774 }
775
776 /**
777  * DPDK callback to close the device.
778  *
779  * @param dev
780  *   Pointer to Ethernet device structure.
781  */
782 static void
783 mrvl_dev_close(struct rte_eth_dev *dev)
784 {
785         struct mrvl_priv *priv = dev->data->dev_private;
786         size_t i;
787
788         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
789                 struct pp2_ppio_tc_params *tc_params =
790                         &priv->ppio_params.inqs_params.tcs_params[i];
791
792                 if (tc_params->inqs_params) {
793                         rte_free(tc_params->inqs_params);
794                         tc_params->inqs_params = NULL;
795                 }
796         }
797
798         mrvl_flush_bpool(dev);
799 }
800
801 /**
802  * DPDK callback to retrieve physical link information.
803  *
804  * @param dev
805  *   Pointer to Ethernet device structure.
806  * @param wait_to_complete
807  *   Wait for request completion (ignored).
808  *
809  * @return
810  *   0 on success, negative error value otherwise.
811  */
812 static int
813 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
814 {
815         /*
816          * TODO
817          * once MUSDK provides necessary API use it here
818          */
819         struct mrvl_priv *priv = dev->data->dev_private;
820         struct ethtool_cmd edata;
821         struct ifreq req;
822         int ret, fd, link_up;
823
824         if (!priv->ppio)
825                 return -EPERM;
826
827         edata.cmd = ETHTOOL_GSET;
828
829         strcpy(req.ifr_name, dev->data->name);
830         req.ifr_data = (void *)&edata;
831
832         fd = socket(AF_INET, SOCK_DGRAM, 0);
833         if (fd == -1)
834                 return -EFAULT;
835
836         ret = ioctl(fd, SIOCETHTOOL, &req);
837         if (ret == -1) {
838                 close(fd);
839                 return -EFAULT;
840         }
841
842         close(fd);
843
844         switch (ethtool_cmd_speed(&edata)) {
845         case SPEED_10:
846                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
847                 break;
848         case SPEED_100:
849                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
850                 break;
851         case SPEED_1000:
852                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
853                 break;
854         case SPEED_10000:
855                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
856                 break;
857         default:
858                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
859         }
860
861         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
862                                                          ETH_LINK_HALF_DUPLEX;
863         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
864                                                            ETH_LINK_FIXED;
865         pp2_ppio_get_link_state(priv->ppio, &link_up);
866         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
867
868         return 0;
869 }
870
871 /**
872  * DPDK callback to enable promiscuous mode.
873  *
874  * @param dev
875  *   Pointer to Ethernet device structure.
876  */
877 static void
878 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
879 {
880         struct mrvl_priv *priv = dev->data->dev_private;
881         int ret;
882
883         if (!priv->ppio)
884                 return;
885
886         if (priv->isolated)
887                 return;
888
889         ret = pp2_ppio_set_promisc(priv->ppio, 1);
890         if (ret)
891                 MRVL_LOG(ERR, "Failed to enable promiscuous mode");
892 }
893
894 /**
895  * DPDK callback to enable allmulti mode.
896  *
897  * @param dev
898  *   Pointer to Ethernet device structure.
899  */
900 static void
901 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
902 {
903         struct mrvl_priv *priv = dev->data->dev_private;
904         int ret;
905
906         if (!priv->ppio)
907                 return;
908
909         if (priv->isolated)
910                 return;
911
912         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
913         if (ret)
914                 MRVL_LOG(ERR, "Failed enable all-multicast mode");
915 }
916
917 /**
918  * DPDK callback to disable promiscuous mode.
919  *
920  * @param dev
921  *   Pointer to Ethernet device structure.
922  */
923 static void
924 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
925 {
926         struct mrvl_priv *priv = dev->data->dev_private;
927         int ret;
928
929         if (!priv->ppio)
930                 return;
931
932         ret = pp2_ppio_set_promisc(priv->ppio, 0);
933         if (ret)
934                 MRVL_LOG(ERR, "Failed to disable promiscuous mode");
935 }
936
937 /**
938  * DPDK callback to disable allmulticast mode.
939  *
940  * @param dev
941  *   Pointer to Ethernet device structure.
942  */
943 static void
944 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
945 {
946         struct mrvl_priv *priv = dev->data->dev_private;
947         int ret;
948
949         if (!priv->ppio)
950                 return;
951
952         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
953         if (ret)
954                 MRVL_LOG(ERR, "Failed to disable all-multicast mode");
955 }
956
957 /**
958  * DPDK callback to remove a MAC address.
959  *
960  * @param dev
961  *   Pointer to Ethernet device structure.
962  * @param index
963  *   MAC address index.
964  */
965 static void
966 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
967 {
968         struct mrvl_priv *priv = dev->data->dev_private;
969         char buf[ETHER_ADDR_FMT_SIZE];
970         int ret;
971
972         if (!priv->ppio)
973                 return;
974
975         if (priv->isolated)
976                 return;
977
978         ret = pp2_ppio_remove_mac_addr(priv->ppio,
979                                        dev->data->mac_addrs[index].addr_bytes);
980         if (ret) {
981                 ether_format_addr(buf, sizeof(buf),
982                                   &dev->data->mac_addrs[index]);
983                 MRVL_LOG(ERR, "Failed to remove mac %s", buf);
984         }
985 }
986
987 /**
988  * DPDK callback to add a MAC address.
989  *
990  * @param dev
991  *   Pointer to Ethernet device structure.
992  * @param mac_addr
993  *   MAC address to register.
994  * @param index
995  *   MAC address index.
996  * @param vmdq
997  *   VMDq pool index to associate address with (unused).
998  *
999  * @return
1000  *   0 on success, negative error value otherwise.
1001  */
1002 static int
1003 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
1004                   uint32_t index, uint32_t vmdq __rte_unused)
1005 {
1006         struct mrvl_priv *priv = dev->data->dev_private;
1007         char buf[ETHER_ADDR_FMT_SIZE];
1008         int ret;
1009
1010         if (priv->isolated)
1011                 return -ENOTSUP;
1012
1013         if (index == 0)
1014                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
1015                 return -1;
1016
1017         if (!priv->ppio)
1018                 return 0;
1019
1020         /*
1021          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
1022          * parameter uc_filter_max. Maximum number of mc addresses is then
1023          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
1024          * 21 respectively.
1025          *
1026          * If more than uc_filter_max uc addresses were added to filter list
1027          * then NIC will switch to promiscuous mode automatically.
1028          *
1029          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
1030          * were added to filter list then NIC will switch to all-multicast mode
1031          * automatically.
1032          */
1033         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
1034         if (ret) {
1035                 ether_format_addr(buf, sizeof(buf), mac_addr);
1036                 MRVL_LOG(ERR, "Failed to add mac %s", buf);
1037                 return -1;
1038         }
1039
1040         return 0;
1041 }
1042
1043 /**
1044  * DPDK callback to set the primary MAC address.
1045  *
1046  * @param dev
1047  *   Pointer to Ethernet device structure.
1048  * @param mac_addr
1049  *   MAC address to register.
1050  *
1051  * @return
1052  *   0 on success, negative error value otherwise.
1053  */
1054 static int
1055 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1056 {
1057         struct mrvl_priv *priv = dev->data->dev_private;
1058         int ret;
1059
1060         if (!priv->ppio)
1061                 return 0;
1062
1063         if (priv->isolated)
1064                 return -ENOTSUP;
1065
1066         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
1067         if (ret) {
1068                 char buf[ETHER_ADDR_FMT_SIZE];
1069                 ether_format_addr(buf, sizeof(buf), mac_addr);
1070                 MRVL_LOG(ERR, "Failed to set mac to %s", buf);
1071         }
1072
1073         return ret;
1074 }
1075
1076 /**
1077  * DPDK callback to get device statistics.
1078  *
1079  * @param dev
1080  *   Pointer to Ethernet device structure.
1081  * @param stats
1082  *   Stats structure output buffer.
1083  *
1084  * @return
1085  *   0 on success, negative error value otherwise.
1086  */
1087 static int
1088 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1089 {
1090         struct mrvl_priv *priv = dev->data->dev_private;
1091         struct pp2_ppio_statistics ppio_stats;
1092         uint64_t drop_mac = 0;
1093         unsigned int i, idx, ret;
1094
1095         if (!priv->ppio)
1096                 return -EPERM;
1097
1098         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1099                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1100                 struct pp2_ppio_inq_statistics rx_stats;
1101
1102                 if (!rxq)
1103                         continue;
1104
1105                 idx = rxq->queue_id;
1106                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1107                         MRVL_LOG(ERR,
1108                                 "rx queue %d stats out of range (0 - %d)",
1109                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1110                         continue;
1111                 }
1112
1113                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
1114                                                   priv->rxq_map[idx].tc,
1115                                                   priv->rxq_map[idx].inq,
1116                                                   &rx_stats, 0);
1117                 if (unlikely(ret)) {
1118                         MRVL_LOG(ERR,
1119                                 "Failed to update rx queue %d stats", idx);
1120                         break;
1121                 }
1122
1123                 stats->q_ibytes[idx] = rxq->bytes_recv;
1124                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
1125                 stats->q_errors[idx] = rx_stats.drop_early +
1126                                        rx_stats.drop_fullq +
1127                                        rx_stats.drop_bm +
1128                                        rxq->drop_mac;
1129                 stats->ibytes += rxq->bytes_recv;
1130                 drop_mac += rxq->drop_mac;
1131         }
1132
1133         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1134                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1135                 struct pp2_ppio_outq_statistics tx_stats;
1136
1137                 if (!txq)
1138                         continue;
1139
1140                 idx = txq->queue_id;
1141                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1142                         MRVL_LOG(ERR,
1143                                 "tx queue %d stats out of range (0 - %d)",
1144                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1145                 }
1146
1147                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1148                                                    &tx_stats, 0);
1149                 if (unlikely(ret)) {
1150                         MRVL_LOG(ERR,
1151                                 "Failed to update tx queue %d stats", idx);
1152                         break;
1153                 }
1154
1155                 stats->q_opackets[idx] = tx_stats.deq_desc;
1156                 stats->q_obytes[idx] = txq->bytes_sent;
1157                 stats->obytes += txq->bytes_sent;
1158         }
1159
1160         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1161         if (unlikely(ret)) {
1162                 MRVL_LOG(ERR, "Failed to update port statistics");
1163                 return ret;
1164         }
1165
1166         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1167         stats->opackets += ppio_stats.tx_packets;
1168         stats->imissed += ppio_stats.rx_fullq_dropped +
1169                           ppio_stats.rx_bm_dropped +
1170                           ppio_stats.rx_early_dropped +
1171                           ppio_stats.rx_fifo_dropped +
1172                           ppio_stats.rx_cls_dropped;
1173         stats->ierrors = drop_mac;
1174
1175         return 0;
1176 }
1177
1178 /**
1179  * DPDK callback to clear device statistics.
1180  *
1181  * @param dev
1182  *   Pointer to Ethernet device structure.
1183  */
1184 static void
1185 mrvl_stats_reset(struct rte_eth_dev *dev)
1186 {
1187         struct mrvl_priv *priv = dev->data->dev_private;
1188         int i;
1189
1190         if (!priv->ppio)
1191                 return;
1192
1193         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1194                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1195
1196                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1197                                             priv->rxq_map[i].inq, NULL, 1);
1198                 rxq->bytes_recv = 0;
1199                 rxq->drop_mac = 0;
1200         }
1201
1202         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1203                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1204
1205                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1206                 txq->bytes_sent = 0;
1207         }
1208
1209         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1210 }
1211
1212 /**
1213  * DPDK callback to get extended statistics.
1214  *
1215  * @param dev
1216  *   Pointer to Ethernet device structure.
1217  * @param stats
1218  *   Pointer to xstats table.
1219  * @param n
1220  *   Number of entries in xstats table.
1221  * @return
1222  *   Negative value on error, number of read xstats otherwise.
1223  */
1224 static int
1225 mrvl_xstats_get(struct rte_eth_dev *dev,
1226                 struct rte_eth_xstat *stats, unsigned int n)
1227 {
1228         struct mrvl_priv *priv = dev->data->dev_private;
1229         struct pp2_ppio_statistics ppio_stats;
1230         unsigned int i;
1231
1232         if (!stats)
1233                 return 0;
1234
1235         pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1236         for (i = 0; i < n && i < RTE_DIM(mrvl_xstats_tbl); i++) {
1237                 uint64_t val;
1238
1239                 if (mrvl_xstats_tbl[i].size == sizeof(uint32_t))
1240                         val = *(uint32_t *)((uint8_t *)&ppio_stats +
1241                                             mrvl_xstats_tbl[i].offset);
1242                 else if (mrvl_xstats_tbl[i].size == sizeof(uint64_t))
1243                         val = *(uint64_t *)((uint8_t *)&ppio_stats +
1244                                             mrvl_xstats_tbl[i].offset);
1245                 else
1246                         return -EINVAL;
1247
1248                 stats[i].id = i;
1249                 stats[i].value = val;
1250         }
1251
1252         return n;
1253 }
1254
1255 /**
1256  * DPDK callback to reset extended statistics.
1257  *
1258  * @param dev
1259  *   Pointer to Ethernet device structure.
1260  */
1261 static void
1262 mrvl_xstats_reset(struct rte_eth_dev *dev)
1263 {
1264         mrvl_stats_reset(dev);
1265 }
1266
1267 /**
1268  * DPDK callback to get extended statistics names.
1269  *
1270  * @param dev (unused)
1271  *   Pointer to Ethernet device structure.
1272  * @param xstats_names
1273  *   Pointer to xstats names table.
1274  * @param size
1275  *   Size of the xstats names table.
1276  * @return
1277  *   Number of read names.
1278  */
1279 static int
1280 mrvl_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
1281                       struct rte_eth_xstat_name *xstats_names,
1282                       unsigned int size)
1283 {
1284         unsigned int i;
1285
1286         if (!xstats_names)
1287                 return RTE_DIM(mrvl_xstats_tbl);
1288
1289         for (i = 0; i < size && i < RTE_DIM(mrvl_xstats_tbl); i++)
1290                 snprintf(xstats_names[i].name, RTE_ETH_XSTATS_NAME_SIZE, "%s",
1291                          mrvl_xstats_tbl[i].name);
1292
1293         return size;
1294 }
1295
1296 /**
1297  * DPDK callback to get information about the device.
1298  *
1299  * @param dev
1300  *   Pointer to Ethernet device structure (unused).
1301  * @param info
1302  *   Info structure output buffer.
1303  */
1304 static void
1305 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
1306                    struct rte_eth_dev_info *info)
1307 {
1308         info->speed_capa = ETH_LINK_SPEED_10M |
1309                            ETH_LINK_SPEED_100M |
1310                            ETH_LINK_SPEED_1G |
1311                            ETH_LINK_SPEED_10G;
1312
1313         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1314         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1315         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1316
1317         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1318         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1319         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1320
1321         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1322         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1323         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1324
1325         info->rx_offload_capa = MRVL_RX_OFFLOADS;
1326         info->rx_queue_offload_capa = MRVL_RX_OFFLOADS;
1327
1328         info->tx_offload_capa = MRVL_TX_OFFLOADS;
1329         info->tx_queue_offload_capa = MRVL_TX_OFFLOADS;
1330
1331         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1332                                        ETH_RSS_NONFRAG_IPV4_TCP |
1333                                        ETH_RSS_NONFRAG_IPV4_UDP;
1334
1335         /* By default packets are dropped if no descriptors are available */
1336         info->default_rxconf.rx_drop_en = 1;
1337         info->default_rxconf.offloads = DEV_RX_OFFLOAD_CRC_STRIP;
1338
1339         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1340 }
1341
1342 /**
1343  * Return supported packet types.
1344  *
1345  * @param dev
1346  *   Pointer to Ethernet device structure (unused).
1347  *
1348  * @return
1349  *   Const pointer to the table with supported packet types.
1350  */
1351 static const uint32_t *
1352 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1353 {
1354         static const uint32_t ptypes[] = {
1355                 RTE_PTYPE_L2_ETHER,
1356                 RTE_PTYPE_L3_IPV4,
1357                 RTE_PTYPE_L3_IPV4_EXT,
1358                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1359                 RTE_PTYPE_L3_IPV6,
1360                 RTE_PTYPE_L3_IPV6_EXT,
1361                 RTE_PTYPE_L2_ETHER_ARP,
1362                 RTE_PTYPE_L4_TCP,
1363                 RTE_PTYPE_L4_UDP
1364         };
1365
1366         return ptypes;
1367 }
1368
1369 /**
1370  * DPDK callback to get information about specific receive queue.
1371  *
1372  * @param dev
1373  *   Pointer to Ethernet device structure.
1374  * @param rx_queue_id
1375  *   Receive queue index.
1376  * @param qinfo
1377  *   Receive queue information structure.
1378  */
1379 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1380                               struct rte_eth_rxq_info *qinfo)
1381 {
1382         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1383         struct mrvl_priv *priv = dev->data->dev_private;
1384         int inq = priv->rxq_map[rx_queue_id].inq;
1385         int tc = priv->rxq_map[rx_queue_id].tc;
1386         struct pp2_ppio_tc_params *tc_params =
1387                 &priv->ppio_params.inqs_params.tcs_params[tc];
1388
1389         qinfo->mp = q->mp;
1390         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1391 }
1392
1393 /**
1394  * DPDK callback to get information about specific transmit queue.
1395  *
1396  * @param dev
1397  *   Pointer to Ethernet device structure.
1398  * @param tx_queue_id
1399  *   Transmit queue index.
1400  * @param qinfo
1401  *   Transmit queue information structure.
1402  */
1403 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1404                               struct rte_eth_txq_info *qinfo)
1405 {
1406         struct mrvl_priv *priv = dev->data->dev_private;
1407         struct mrvl_txq *txq = dev->data->tx_queues[tx_queue_id];
1408
1409         qinfo->nb_desc =
1410                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1411         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1412 }
1413
1414 /**
1415  * DPDK callback to Configure a VLAN filter.
1416  *
1417  * @param dev
1418  *   Pointer to Ethernet device structure.
1419  * @param vlan_id
1420  *   VLAN ID to filter.
1421  * @param on
1422  *   Toggle filter.
1423  *
1424  * @return
1425  *   0 on success, negative error value otherwise.
1426  */
1427 static int
1428 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1429 {
1430         struct mrvl_priv *priv = dev->data->dev_private;
1431
1432         if (!priv->ppio)
1433                 return -EPERM;
1434
1435         if (priv->isolated)
1436                 return -ENOTSUP;
1437
1438         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1439                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1440 }
1441
1442 /**
1443  * Release buffers to hardware bpool (buffer-pool)
1444  *
1445  * @param rxq
1446  *   Receive queue pointer.
1447  * @param num
1448  *   Number of buffers to release to bpool.
1449  *
1450  * @return
1451  *   0 on success, negative error value otherwise.
1452  */
1453 static int
1454 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1455 {
1456         struct buff_release_entry entries[MRVL_PP2_RXD_MAX];
1457         struct rte_mbuf *mbufs[MRVL_PP2_RXD_MAX];
1458         int i, ret;
1459         unsigned int core_id;
1460         struct pp2_hif *hif;
1461         struct pp2_bpool *bpool;
1462
1463         core_id = rte_lcore_id();
1464         if (core_id == LCORE_ID_ANY)
1465                 core_id = 0;
1466
1467         hif = mrvl_get_hif(rxq->priv, core_id);
1468         if (!hif)
1469                 return -1;
1470
1471         bpool = rxq->priv->bpool;
1472
1473         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1474         if (ret)
1475                 return ret;
1476
1477         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1478                 cookie_addr_high =
1479                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1480
1481         for (i = 0; i < num; i++) {
1482                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1483                         != cookie_addr_high) {
1484                         MRVL_LOG(ERR,
1485                                 "mbuf virtual addr high 0x%lx out of range",
1486                                 (uint64_t)mbufs[i] >> 32);
1487                         goto out;
1488                 }
1489
1490                 entries[i].buff.addr =
1491                         rte_mbuf_data_iova_default(mbufs[i]);
1492                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
1493                 entries[i].bpool = bpool;
1494         }
1495
1496         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1497         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1498
1499         if (i != num)
1500                 goto out;
1501
1502         return 0;
1503 out:
1504         for (; i < num; i++)
1505                 rte_pktmbuf_free(mbufs[i]);
1506
1507         return -1;
1508 }
1509
1510 /**
1511  * DPDK callback to configure the receive queue.
1512  *
1513  * @param dev
1514  *   Pointer to Ethernet device structure.
1515  * @param idx
1516  *   RX queue index.
1517  * @param desc
1518  *   Number of descriptors to configure in queue.
1519  * @param socket
1520  *   NUMA socket on which memory must be allocated.
1521  * @param conf
1522  *   Thresholds parameters.
1523  * @param mp
1524  *   Memory pool for buffer allocations.
1525  *
1526  * @return
1527  *   0 on success, negative error value otherwise.
1528  */
1529 static int
1530 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1531                     unsigned int socket,
1532                     const struct rte_eth_rxconf *conf,
1533                     struct rte_mempool *mp)
1534 {
1535         struct mrvl_priv *priv = dev->data->dev_private;
1536         struct mrvl_rxq *rxq;
1537         uint32_t min_size,
1538                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1539         int ret, tc, inq;
1540         uint64_t offloads;
1541
1542         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1543
1544         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1545                 /*
1546                  * Unknown TC mapping, mapping will not have a correct queue.
1547                  */
1548                 MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu",
1549                         idx, priv->ppio_id);
1550                 return -EFAULT;
1551         }
1552
1553         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
1554                    MRVL_PKT_EFFEC_OFFS;
1555         if (min_size < max_rx_pkt_len) {
1556                 MRVL_LOG(ERR,
1557                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.",
1558                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
1559                         MRVL_PKT_EFFEC_OFFS,
1560                         max_rx_pkt_len);
1561                 return -EINVAL;
1562         }
1563
1564         if (dev->data->rx_queues[idx]) {
1565                 rte_free(dev->data->rx_queues[idx]);
1566                 dev->data->rx_queues[idx] = NULL;
1567         }
1568
1569         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1570         if (!rxq)
1571                 return -ENOMEM;
1572
1573         rxq->priv = priv;
1574         rxq->mp = mp;
1575         rxq->cksum_enabled = offloads & DEV_RX_OFFLOAD_IPV4_CKSUM;
1576         rxq->queue_id = idx;
1577         rxq->port_id = dev->data->port_id;
1578         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1579
1580         tc = priv->rxq_map[rxq->queue_id].tc,
1581         inq = priv->rxq_map[rxq->queue_id].inq;
1582         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1583                 desc;
1584
1585         ret = mrvl_fill_bpool(rxq, desc);
1586         if (ret) {
1587                 rte_free(rxq);
1588                 return ret;
1589         }
1590
1591         priv->bpool_init_size += desc;
1592
1593         dev->data->rx_queues[idx] = rxq;
1594
1595         return 0;
1596 }
1597
1598 /**
1599  * DPDK callback to release the receive queue.
1600  *
1601  * @param rxq
1602  *   Generic receive queue pointer.
1603  */
1604 static void
1605 mrvl_rx_queue_release(void *rxq)
1606 {
1607         struct mrvl_rxq *q = rxq;
1608         struct pp2_ppio_tc_params *tc_params;
1609         int i, num, tc, inq;
1610         struct pp2_hif *hif;
1611         unsigned int core_id = rte_lcore_id();
1612
1613         if (core_id == LCORE_ID_ANY)
1614                 core_id = 0;
1615
1616         if (!q)
1617                 return;
1618
1619         hif = mrvl_get_hif(q->priv, core_id);
1620
1621         if (!hif)
1622                 return;
1623
1624         tc = q->priv->rxq_map[q->queue_id].tc;
1625         inq = q->priv->rxq_map[q->queue_id].inq;
1626         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1627         num = tc_params->inqs_params[inq].size;
1628         for (i = 0; i < num; i++) {
1629                 struct pp2_buff_inf inf;
1630                 uint64_t addr;
1631
1632                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
1633                 addr = cookie_addr_high | inf.cookie;
1634                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1635         }
1636
1637         rte_free(q);
1638 }
1639
1640 /**
1641  * DPDK callback to configure the transmit queue.
1642  *
1643  * @param dev
1644  *   Pointer to Ethernet device structure.
1645  * @param idx
1646  *   Transmit queue index.
1647  * @param desc
1648  *   Number of descriptors to configure in the queue.
1649  * @param socket
1650  *   NUMA socket on which memory must be allocated.
1651  * @param conf
1652  *   Tx queue configuration parameters.
1653  *
1654  * @return
1655  *   0 on success, negative error value otherwise.
1656  */
1657 static int
1658 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1659                     unsigned int socket,
1660                     const struct rte_eth_txconf *conf)
1661 {
1662         struct mrvl_priv *priv = dev->data->dev_private;
1663         struct mrvl_txq *txq;
1664
1665         if (dev->data->tx_queues[idx]) {
1666                 rte_free(dev->data->tx_queues[idx]);
1667                 dev->data->tx_queues[idx] = NULL;
1668         }
1669
1670         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1671         if (!txq)
1672                 return -ENOMEM;
1673
1674         txq->priv = priv;
1675         txq->queue_id = idx;
1676         txq->port_id = dev->data->port_id;
1677         txq->tx_deferred_start = conf->tx_deferred_start;
1678         dev->data->tx_queues[idx] = txq;
1679
1680         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1681
1682         return 0;
1683 }
1684
1685 /**
1686  * DPDK callback to release the transmit queue.
1687  *
1688  * @param txq
1689  *   Generic transmit queue pointer.
1690  */
1691 static void
1692 mrvl_tx_queue_release(void *txq)
1693 {
1694         struct mrvl_txq *q = txq;
1695
1696         if (!q)
1697                 return;
1698
1699         rte_free(q);
1700 }
1701
1702 /**
1703  * DPDK callback to get flow control configuration.
1704  *
1705  * @param dev
1706  *  Pointer to Ethernet device structure.
1707  * @param fc_conf
1708  *  Pointer to the flow control configuration.
1709  *
1710  * @return
1711  *  0 on success, negative error value otherwise.
1712  */
1713 static int
1714 mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1715 {
1716         struct mrvl_priv *priv = dev->data->dev_private;
1717         int ret, en;
1718
1719         if (!priv)
1720                 return -EPERM;
1721
1722         ret = pp2_ppio_get_rx_pause(priv->ppio, &en);
1723         if (ret) {
1724                 MRVL_LOG(ERR, "Failed to read rx pause state");
1725                 return ret;
1726         }
1727
1728         fc_conf->mode = en ? RTE_FC_RX_PAUSE : RTE_FC_NONE;
1729
1730         return 0;
1731 }
1732
1733 /**
1734  * DPDK callback to set flow control configuration.
1735  *
1736  * @param dev
1737  *  Pointer to Ethernet device structure.
1738  * @param fc_conf
1739  *  Pointer to the flow control configuration.
1740  *
1741  * @return
1742  *  0 on success, negative error value otherwise.
1743  */
1744 static int
1745 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1746 {
1747         struct mrvl_priv *priv = dev->data->dev_private;
1748
1749         if (!priv)
1750                 return -EPERM;
1751
1752         if (fc_conf->high_water ||
1753             fc_conf->low_water ||
1754             fc_conf->pause_time ||
1755             fc_conf->mac_ctrl_frame_fwd ||
1756             fc_conf->autoneg) {
1757                 MRVL_LOG(ERR, "Flowctrl parameter is not supported");
1758
1759                 return -EINVAL;
1760         }
1761
1762         if (fc_conf->mode == RTE_FC_NONE ||
1763             fc_conf->mode == RTE_FC_RX_PAUSE) {
1764                 int ret, en;
1765
1766                 en = fc_conf->mode == RTE_FC_NONE ? 0 : 1;
1767                 ret = pp2_ppio_set_rx_pause(priv->ppio, en);
1768                 if (ret)
1769                         MRVL_LOG(ERR,
1770                                 "Failed to change flowctrl on RX side");
1771
1772                 return ret;
1773         }
1774
1775         return 0;
1776 }
1777
1778 /**
1779  * Update RSS hash configuration
1780  *
1781  * @param dev
1782  *   Pointer to Ethernet device structure.
1783  * @param rss_conf
1784  *   Pointer to RSS configuration.
1785  *
1786  * @return
1787  *   0 on success, negative error value otherwise.
1788  */
1789 static int
1790 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1791                      struct rte_eth_rss_conf *rss_conf)
1792 {
1793         struct mrvl_priv *priv = dev->data->dev_private;
1794
1795         if (priv->isolated)
1796                 return -ENOTSUP;
1797
1798         return mrvl_configure_rss(priv, rss_conf);
1799 }
1800
1801 /**
1802  * DPDK callback to get RSS hash configuration.
1803  *
1804  * @param dev
1805  *   Pointer to Ethernet device structure.
1806  * @rss_conf
1807  *   Pointer to RSS configuration.
1808  *
1809  * @return
1810  *   Always 0.
1811  */
1812 static int
1813 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1814                        struct rte_eth_rss_conf *rss_conf)
1815 {
1816         struct mrvl_priv *priv = dev->data->dev_private;
1817         enum pp2_ppio_hash_type hash_type =
1818                 priv->ppio_params.inqs_params.hash_type;
1819
1820         rss_conf->rss_key = NULL;
1821
1822         if (hash_type == PP2_PPIO_HASH_T_NONE)
1823                 rss_conf->rss_hf = 0;
1824         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1825                 rss_conf->rss_hf = ETH_RSS_IPV4;
1826         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1827                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1828         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1829                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1830
1831         return 0;
1832 }
1833
1834 /**
1835  * DPDK callback to get rte_flow callbacks.
1836  *
1837  * @param dev
1838  *   Pointer to the device structure.
1839  * @param filer_type
1840  *   Flow filter type.
1841  * @param filter_op
1842  *   Flow filter operation.
1843  * @param arg
1844  *   Pointer to pass the flow ops.
1845  *
1846  * @return
1847  *   0 on success, negative error value otherwise.
1848  */
1849 static int
1850 mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
1851                      enum rte_filter_type filter_type,
1852                      enum rte_filter_op filter_op, void *arg)
1853 {
1854         switch (filter_type) {
1855         case RTE_ETH_FILTER_GENERIC:
1856                 if (filter_op != RTE_ETH_FILTER_GET)
1857                         return -EINVAL;
1858                 *(const void **)arg = &mrvl_flow_ops;
1859                 return 0;
1860         default:
1861                 MRVL_LOG(WARNING, "Filter type (%d) not supported",
1862                                 filter_type);
1863                 return -EINVAL;
1864         }
1865 }
1866
1867 static const struct eth_dev_ops mrvl_ops = {
1868         .dev_configure = mrvl_dev_configure,
1869         .dev_start = mrvl_dev_start,
1870         .dev_stop = mrvl_dev_stop,
1871         .dev_set_link_up = mrvl_dev_set_link_up,
1872         .dev_set_link_down = mrvl_dev_set_link_down,
1873         .dev_close = mrvl_dev_close,
1874         .link_update = mrvl_link_update,
1875         .promiscuous_enable = mrvl_promiscuous_enable,
1876         .allmulticast_enable = mrvl_allmulticast_enable,
1877         .promiscuous_disable = mrvl_promiscuous_disable,
1878         .allmulticast_disable = mrvl_allmulticast_disable,
1879         .mac_addr_remove = mrvl_mac_addr_remove,
1880         .mac_addr_add = mrvl_mac_addr_add,
1881         .mac_addr_set = mrvl_mac_addr_set,
1882         .mtu_set = mrvl_mtu_set,
1883         .stats_get = mrvl_stats_get,
1884         .stats_reset = mrvl_stats_reset,
1885         .xstats_get = mrvl_xstats_get,
1886         .xstats_reset = mrvl_xstats_reset,
1887         .xstats_get_names = mrvl_xstats_get_names,
1888         .dev_infos_get = mrvl_dev_infos_get,
1889         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
1890         .rxq_info_get = mrvl_rxq_info_get,
1891         .txq_info_get = mrvl_txq_info_get,
1892         .vlan_filter_set = mrvl_vlan_filter_set,
1893         .tx_queue_start = mrvl_tx_queue_start,
1894         .tx_queue_stop = mrvl_tx_queue_stop,
1895         .rx_queue_setup = mrvl_rx_queue_setup,
1896         .rx_queue_release = mrvl_rx_queue_release,
1897         .tx_queue_setup = mrvl_tx_queue_setup,
1898         .tx_queue_release = mrvl_tx_queue_release,
1899         .flow_ctrl_get = mrvl_flow_ctrl_get,
1900         .flow_ctrl_set = mrvl_flow_ctrl_set,
1901         .rss_hash_update = mrvl_rss_hash_update,
1902         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1903         .filter_ctrl = mrvl_eth_filter_ctrl,
1904 };
1905
1906 /**
1907  * Return packet type information and l3/l4 offsets.
1908  *
1909  * @param desc
1910  *   Pointer to the received packet descriptor.
1911  * @param l3_offset
1912  *   l3 packet offset.
1913  * @param l4_offset
1914  *   l4 packet offset.
1915  *
1916  * @return
1917  *   Packet type information.
1918  */
1919 static inline uint64_t
1920 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
1921                                     uint8_t *l3_offset, uint8_t *l4_offset)
1922 {
1923         enum pp2_inq_l3_type l3_type;
1924         enum pp2_inq_l4_type l4_type;
1925         uint64_t packet_type;
1926
1927         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
1928         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
1929
1930         packet_type = RTE_PTYPE_L2_ETHER;
1931
1932         switch (l3_type) {
1933         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
1934                 packet_type |= RTE_PTYPE_L3_IPV4;
1935                 break;
1936         case PP2_INQ_L3_TYPE_IPV4_OK:
1937                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
1938                 break;
1939         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
1940                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
1941                 break;
1942         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
1943                 packet_type |= RTE_PTYPE_L3_IPV6;
1944                 break;
1945         case PP2_INQ_L3_TYPE_IPV6_EXT:
1946                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
1947                 break;
1948         case PP2_INQ_L3_TYPE_ARP:
1949                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
1950                 /*
1951                  * In case of ARP l4_offset is set to wrong value.
1952                  * Set it to proper one so that later on mbuf->l3_len can be
1953                  * calculated subtracting l4_offset and l3_offset.
1954                  */
1955                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
1956                 break;
1957         default:
1958                 MRVL_LOG(DEBUG, "Failed to recognise l3 packet type");
1959                 break;
1960         }
1961
1962         switch (l4_type) {
1963         case PP2_INQ_L4_TYPE_TCP:
1964                 packet_type |= RTE_PTYPE_L4_TCP;
1965                 break;
1966         case PP2_INQ_L4_TYPE_UDP:
1967                 packet_type |= RTE_PTYPE_L4_UDP;
1968                 break;
1969         default:
1970                 MRVL_LOG(DEBUG, "Failed to recognise l4 packet type");
1971                 break;
1972         }
1973
1974         return packet_type;
1975 }
1976
1977 /**
1978  * Get offload information from the received packet descriptor.
1979  *
1980  * @param desc
1981  *   Pointer to the received packet descriptor.
1982  *
1983  * @return
1984  *   Mbuf offload flags.
1985  */
1986 static inline uint64_t
1987 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
1988 {
1989         uint64_t flags;
1990         enum pp2_inq_desc_status status;
1991
1992         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
1993         if (unlikely(status != PP2_DESC_ERR_OK))
1994                 flags = PKT_RX_IP_CKSUM_BAD;
1995         else
1996                 flags = PKT_RX_IP_CKSUM_GOOD;
1997
1998         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
1999         if (unlikely(status != PP2_DESC_ERR_OK))
2000                 flags |= PKT_RX_L4_CKSUM_BAD;
2001         else
2002                 flags |= PKT_RX_L4_CKSUM_GOOD;
2003
2004         return flags;
2005 }
2006
2007 /**
2008  * DPDK callback for receive.
2009  *
2010  * @param rxq
2011  *   Generic pointer to the receive queue.
2012  * @param rx_pkts
2013  *   Array to store received packets.
2014  * @param nb_pkts
2015  *   Maximum number of packets in array.
2016  *
2017  * @return
2018  *   Number of packets successfully received.
2019  */
2020 static uint16_t
2021 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2022 {
2023         struct mrvl_rxq *q = rxq;
2024         struct pp2_ppio_desc descs[nb_pkts];
2025         struct pp2_bpool *bpool;
2026         int i, ret, rx_done = 0;
2027         int num;
2028         struct pp2_hif *hif;
2029         unsigned int core_id = rte_lcore_id();
2030
2031         hif = mrvl_get_hif(q->priv, core_id);
2032
2033         if (unlikely(!q->priv->ppio || !hif))
2034                 return 0;
2035
2036         bpool = q->priv->bpool;
2037
2038         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
2039                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
2040         if (unlikely(ret < 0)) {
2041                 MRVL_LOG(ERR, "Failed to receive packets");
2042                 return 0;
2043         }
2044         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
2045
2046         for (i = 0; i < nb_pkts; i++) {
2047                 struct rte_mbuf *mbuf;
2048                 uint8_t l3_offset, l4_offset;
2049                 enum pp2_inq_desc_status status;
2050                 uint64_t addr;
2051
2052                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2053                         struct pp2_ppio_desc *pref_desc;
2054                         u64 pref_addr;
2055
2056                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
2057                         pref_addr = cookie_addr_high |
2058                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
2059                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
2060                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
2061                 }
2062
2063                 addr = cookie_addr_high |
2064                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
2065                 mbuf = (struct rte_mbuf *)addr;
2066                 rte_pktmbuf_reset(mbuf);
2067
2068                 /* drop packet in case of mac, overrun or resource error */
2069                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
2070                 if (unlikely(status != PP2_DESC_ERR_OK)) {
2071                         struct pp2_buff_inf binf = {
2072                                 .addr = rte_mbuf_data_iova_default(mbuf),
2073                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
2074                         };
2075
2076                         pp2_bpool_put_buff(hif, bpool, &binf);
2077                         mrvl_port_bpool_size
2078                                 [bpool->pp2_id][bpool->id][core_id]++;
2079                         q->drop_mac++;
2080                         continue;
2081                 }
2082
2083                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
2084                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
2085                 mbuf->data_len = mbuf->pkt_len;
2086                 mbuf->port = q->port_id;
2087                 mbuf->packet_type =
2088                         mrvl_desc_to_packet_type_and_offset(&descs[i],
2089                                                             &l3_offset,
2090                                                             &l4_offset);
2091                 mbuf->l2_len = l3_offset;
2092                 mbuf->l3_len = l4_offset - l3_offset;
2093
2094                 if (likely(q->cksum_enabled))
2095                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
2096
2097                 rx_pkts[rx_done++] = mbuf;
2098                 q->bytes_recv += mbuf->pkt_len;
2099         }
2100
2101         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
2102                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
2103
2104                 if (unlikely(num <= q->priv->bpool_min_size ||
2105                              (!rx_done && num < q->priv->bpool_init_size))) {
2106                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
2107                         if (ret)
2108                                 MRVL_LOG(ERR, "Failed to fill bpool");
2109                 } else if (unlikely(num > q->priv->bpool_max_size)) {
2110                         int i;
2111                         int pkt_to_remove = num - q->priv->bpool_init_size;
2112                         struct rte_mbuf *mbuf;
2113                         struct pp2_buff_inf buff;
2114
2115                         MRVL_LOG(DEBUG,
2116                                 "port-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)",
2117                                 bpool->pp2_id, q->priv->ppio->port_id,
2118                                 bpool->id, pkt_to_remove, num,
2119                                 q->priv->bpool_init_size);
2120
2121                         for (i = 0; i < pkt_to_remove; i++) {
2122                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
2123                                 if (ret)
2124                                         break;
2125                                 mbuf = (struct rte_mbuf *)
2126                                         (cookie_addr_high | buff.cookie);
2127                                 rte_pktmbuf_free(mbuf);
2128                         }
2129                         mrvl_port_bpool_size
2130                                 [bpool->pp2_id][bpool->id][core_id] -= i;
2131                 }
2132                 rte_spinlock_unlock(&q->priv->lock);
2133         }
2134
2135         return rx_done;
2136 }
2137
2138 /**
2139  * Prepare offload information.
2140  *
2141  * @param ol_flags
2142  *   Offload flags.
2143  * @param packet_type
2144  *   Packet type bitfield.
2145  * @param l3_type
2146  *   Pointer to the pp2_ouq_l3_type structure.
2147  * @param l4_type
2148  *   Pointer to the pp2_outq_l4_type structure.
2149  * @param gen_l3_cksum
2150  *   Will be set to 1 in case l3 checksum is computed.
2151  * @param l4_cksum
2152  *   Will be set to 1 in case l4 checksum is computed.
2153  *
2154  * @return
2155  *   0 on success, negative error value otherwise.
2156  */
2157 static inline int
2158 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
2159                         enum pp2_outq_l3_type *l3_type,
2160                         enum pp2_outq_l4_type *l4_type,
2161                         int *gen_l3_cksum,
2162                         int *gen_l4_cksum)
2163 {
2164         /*
2165          * Based on ol_flags prepare information
2166          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
2167          * for offloading.
2168          */
2169         if (ol_flags & PKT_TX_IPV4) {
2170                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
2171                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
2172         } else if (ol_flags & PKT_TX_IPV6) {
2173                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
2174                 /* no checksum for ipv6 header */
2175                 *gen_l3_cksum = 0;
2176         } else {
2177                 /* if something different then stop processing */
2178                 return -1;
2179         }
2180
2181         ol_flags &= PKT_TX_L4_MASK;
2182         if ((packet_type & RTE_PTYPE_L4_TCP) &&
2183             ol_flags == PKT_TX_TCP_CKSUM) {
2184                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
2185                 *gen_l4_cksum = 1;
2186         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
2187                    ol_flags == PKT_TX_UDP_CKSUM) {
2188                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
2189                 *gen_l4_cksum = 1;
2190         } else {
2191                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
2192                 /* no checksum for other type */
2193                 *gen_l4_cksum = 0;
2194         }
2195
2196         return 0;
2197 }
2198
2199 /**
2200  * Release already sent buffers to bpool (buffer-pool).
2201  *
2202  * @param ppio
2203  *   Pointer to the port structure.
2204  * @param hif
2205  *   Pointer to the MUSDK hardware interface.
2206  * @param sq
2207  *   Pointer to the shadow queue.
2208  * @param qid
2209  *   Queue id number.
2210  * @param force
2211  *   Force releasing packets.
2212  */
2213 static inline void
2214 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
2215                        unsigned int core_id, struct mrvl_shadow_txq *sq,
2216                        int qid, int force)
2217 {
2218         struct buff_release_entry *entry;
2219         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
2220         int i;
2221
2222         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
2223
2224         sq->num_to_release += nb_done;
2225
2226         if (likely(!force &&
2227                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
2228                 return;
2229
2230         nb_done = sq->num_to_release;
2231         sq->num_to_release = 0;
2232
2233         for (i = 0; i < nb_done; i++) {
2234                 entry = &sq->ent[sq->tail + num];
2235                 if (unlikely(!entry->buff.addr)) {
2236                         MRVL_LOG(ERR,
2237                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!",
2238                                 sq->tail, (u64)entry->buff.cookie,
2239                                 (u64)entry->buff.addr);
2240                         skip_bufs = 1;
2241                         goto skip;
2242                 }
2243
2244                 if (unlikely(!entry->bpool)) {
2245                         struct rte_mbuf *mbuf;
2246
2247                         mbuf = (struct rte_mbuf *)
2248                                (cookie_addr_high | entry->buff.cookie);
2249                         rte_pktmbuf_free(mbuf);
2250                         skip_bufs = 1;
2251                         goto skip;
2252                 }
2253
2254                 mrvl_port_bpool_size
2255                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
2256                 num++;
2257                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
2258                         goto skip;
2259                 continue;
2260 skip:
2261                 if (likely(num))
2262                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2263                 num += skip_bufs;
2264                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2265                 sq->size -= num;
2266                 num = 0;
2267                 skip_bufs = 0;
2268         }
2269
2270         if (likely(num)) {
2271                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2272                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2273                 sq->size -= num;
2274         }
2275 }
2276
2277 /**
2278  * DPDK callback for transmit.
2279  *
2280  * @param txq
2281  *   Generic pointer transmit queue.
2282  * @param tx_pkts
2283  *   Packets to transmit.
2284  * @param nb_pkts
2285  *   Number of packets in array.
2286  *
2287  * @return
2288  *   Number of packets successfully transmitted.
2289  */
2290 static uint16_t
2291 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2292 {
2293         struct mrvl_txq *q = txq;
2294         struct mrvl_shadow_txq *sq;
2295         struct pp2_hif *hif;
2296         struct pp2_ppio_desc descs[nb_pkts];
2297         unsigned int core_id = rte_lcore_id();
2298         int i, ret, bytes_sent = 0;
2299         uint16_t num, sq_free_size;
2300         uint64_t addr;
2301
2302         hif = mrvl_get_hif(q->priv, core_id);
2303         sq = &q->shadow_txqs[core_id];
2304
2305         if (unlikely(!q->priv->ppio || !hif))
2306                 return 0;
2307
2308         if (sq->size)
2309                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2310                                        sq, q->queue_id, 0);
2311
2312         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2313         if (unlikely(nb_pkts > sq_free_size)) {
2314                 MRVL_LOG(DEBUG,
2315                         "No room in shadow queue for %d packets! %d packets will be sent.",
2316                         nb_pkts, sq_free_size);
2317                 nb_pkts = sq_free_size;
2318         }
2319
2320         for (i = 0; i < nb_pkts; i++) {
2321                 struct rte_mbuf *mbuf = tx_pkts[i];
2322                 int gen_l3_cksum, gen_l4_cksum;
2323                 enum pp2_outq_l3_type l3_type;
2324                 enum pp2_outq_l4_type l4_type;
2325
2326                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2327                         struct rte_mbuf *pref_pkt_hdr;
2328
2329                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2330                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2331                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2332                 }
2333
2334                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
2335                 sq->ent[sq->head].buff.addr =
2336                         rte_mbuf_data_iova_default(mbuf);
2337                 sq->ent[sq->head].bpool =
2338                         (unlikely(mbuf->port >= RTE_MAX_ETHPORTS ||
2339                          mbuf->refcnt > 1)) ? NULL :
2340                          mrvl_port_to_bpool_lookup[mbuf->port];
2341                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
2342                 sq->size++;
2343
2344                 pp2_ppio_outq_desc_reset(&descs[i]);
2345                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
2346                                                  rte_pktmbuf_iova(mbuf));
2347                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
2348                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
2349                                                rte_pktmbuf_pkt_len(mbuf));
2350
2351                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2352                 /*
2353                  * in case unsupported ol_flags were passed
2354                  * do not update descriptor offload information
2355                  */
2356                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
2357                                               &l3_type, &l4_type, &gen_l3_cksum,
2358                                               &gen_l4_cksum);
2359                 if (unlikely(ret))
2360                         continue;
2361
2362                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2363                                                   mbuf->l2_len,
2364                                                   mbuf->l2_len + mbuf->l3_len,
2365                                                   gen_l3_cksum, gen_l4_cksum);
2366         }
2367
2368         num = nb_pkts;
2369         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2370         /* number of packets that were not sent */
2371         if (unlikely(num > nb_pkts)) {
2372                 for (i = nb_pkts; i < num; i++) {
2373                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2374                                 MRVL_PP2_TX_SHADOWQ_MASK;
2375                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
2376                         bytes_sent -=
2377                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2378                 }
2379                 sq->size -= num - nb_pkts;
2380         }
2381
2382         q->bytes_sent += bytes_sent;
2383
2384         return nb_pkts;
2385 }
2386
2387 /**
2388  * Initialize packet processor.
2389  *
2390  * @return
2391  *   0 on success, negative error value otherwise.
2392  */
2393 static int
2394 mrvl_init_pp2(void)
2395 {
2396         struct pp2_init_params init_params;
2397
2398         memset(&init_params, 0, sizeof(init_params));
2399         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
2400         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
2401         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
2402
2403         return pp2_init(&init_params);
2404 }
2405
2406 /**
2407  * Deinitialize packet processor.
2408  *
2409  * @return
2410  *   0 on success, negative error value otherwise.
2411  */
2412 static void
2413 mrvl_deinit_pp2(void)
2414 {
2415         pp2_deinit();
2416 }
2417
2418 /**
2419  * Create private device structure.
2420  *
2421  * @param dev_name
2422  *   Pointer to the port name passed in the initialization parameters.
2423  *
2424  * @return
2425  *   Pointer to the newly allocated private device structure.
2426  */
2427 static struct mrvl_priv *
2428 mrvl_priv_create(const char *dev_name)
2429 {
2430         struct pp2_bpool_params bpool_params;
2431         char match[MRVL_MATCH_LEN];
2432         struct mrvl_priv *priv;
2433         int ret, bpool_bit;
2434
2435         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
2436         if (!priv)
2437                 return NULL;
2438
2439         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
2440                                        &priv->pp_id, &priv->ppio_id);
2441         if (ret)
2442                 goto out_free_priv;
2443
2444         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
2445                                      PP2_BPOOL_NUM_POOLS);
2446         if (bpool_bit < 0)
2447                 goto out_free_priv;
2448         priv->bpool_bit = bpool_bit;
2449
2450         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
2451                  priv->bpool_bit);
2452         memset(&bpool_params, 0, sizeof(bpool_params));
2453         bpool_params.match = match;
2454         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
2455         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
2456         if (ret)
2457                 goto out_clear_bpool_bit;
2458
2459         priv->ppio_params.type = PP2_PPIO_T_NIC;
2460         rte_spinlock_init(&priv->lock);
2461
2462         return priv;
2463 out_clear_bpool_bit:
2464         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2465 out_free_priv:
2466         rte_free(priv);
2467         return NULL;
2468 }
2469
2470 /**
2471  * Create device representing Ethernet port.
2472  *
2473  * @param name
2474  *   Pointer to the port's name.
2475  *
2476  * @return
2477  *   0 on success, negative error value otherwise.
2478  */
2479 static int
2480 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
2481 {
2482         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
2483         struct rte_eth_dev *eth_dev;
2484         struct mrvl_priv *priv;
2485         struct ifreq req;
2486
2487         eth_dev = rte_eth_dev_allocate(name);
2488         if (!eth_dev)
2489                 return -ENOMEM;
2490
2491         priv = mrvl_priv_create(name);
2492         if (!priv) {
2493                 ret = -ENOMEM;
2494                 goto out_free_dev;
2495         }
2496
2497         eth_dev->data->mac_addrs =
2498                 rte_zmalloc("mac_addrs",
2499                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2500         if (!eth_dev->data->mac_addrs) {
2501                 MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
2502                 ret = -ENOMEM;
2503                 goto out_free_priv;
2504         }
2505
2506         memset(&req, 0, sizeof(req));
2507         strcpy(req.ifr_name, name);
2508         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2509         if (ret)
2510                 goto out_free_mac;
2511
2512         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2513                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2514
2515         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2516         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
2517         eth_dev->data->kdrv = RTE_KDRV_NONE;
2518         eth_dev->data->dev_private = priv;
2519         eth_dev->device = &vdev->device;
2520         eth_dev->dev_ops = &mrvl_ops;
2521
2522         rte_eth_dev_probing_finish(eth_dev);
2523         return 0;
2524 out_free_mac:
2525         rte_free(eth_dev->data->mac_addrs);
2526 out_free_dev:
2527         rte_eth_dev_release_port(eth_dev);
2528 out_free_priv:
2529         rte_free(priv);
2530
2531         return ret;
2532 }
2533
2534 /**
2535  * Cleanup previously created device representing Ethernet port.
2536  *
2537  * @param name
2538  *   Pointer to the port name.
2539  */
2540 static void
2541 mrvl_eth_dev_destroy(const char *name)
2542 {
2543         struct rte_eth_dev *eth_dev;
2544         struct mrvl_priv *priv;
2545
2546         eth_dev = rte_eth_dev_allocated(name);
2547         if (!eth_dev)
2548                 return;
2549
2550         priv = eth_dev->data->dev_private;
2551         pp2_bpool_deinit(priv->bpool);
2552         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2553         rte_free(priv);
2554         rte_free(eth_dev->data->mac_addrs);
2555         rte_eth_dev_release_port(eth_dev);
2556 }
2557
2558 /**
2559  * Callback used by rte_kvargs_process() during argument parsing.
2560  *
2561  * @param key
2562  *   Pointer to the parsed key (unused).
2563  * @param value
2564  *   Pointer to the parsed value.
2565  * @param extra_args
2566  *   Pointer to the extra arguments which contains address of the
2567  *   table of pointers to parsed interface names.
2568  *
2569  * @return
2570  *   Always 0.
2571  */
2572 static int
2573 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2574                  void *extra_args)
2575 {
2576         struct mrvl_ifnames *ifnames = extra_args;
2577
2578         ifnames->names[ifnames->idx++] = value;
2579
2580         return 0;
2581 }
2582
2583 /**
2584  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2585  */
2586 static void
2587 mrvl_deinit_hifs(void)
2588 {
2589         int i;
2590
2591         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
2592                 if (hifs[i])
2593                         pp2_hif_deinit(hifs[i]);
2594         }
2595         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
2596         memset(hifs, 0, sizeof(hifs));
2597 }
2598
2599 /**
2600  * DPDK callback to register the virtual device.
2601  *
2602  * @param vdev
2603  *   Pointer to the virtual device.
2604  *
2605  * @return
2606  *   0 on success, negative error value otherwise.
2607  */
2608 static int
2609 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2610 {
2611         struct rte_kvargs *kvlist;
2612         struct mrvl_ifnames ifnames;
2613         int ret = -EINVAL;
2614         uint32_t i, ifnum, cfgnum;
2615         const char *params;
2616
2617         params = rte_vdev_device_args(vdev);
2618         if (!params)
2619                 return -EINVAL;
2620
2621         kvlist = rte_kvargs_parse(params, valid_args);
2622         if (!kvlist)
2623                 return -EINVAL;
2624
2625         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2626         if (ifnum > RTE_DIM(ifnames.names))
2627                 goto out_free_kvlist;
2628
2629         ifnames.idx = 0;
2630         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2631                            mrvl_get_ifnames, &ifnames);
2632
2633
2634         /*
2635          * The below system initialization should be done only once,
2636          * on the first provided configuration file
2637          */
2638         if (!mrvl_qos_cfg) {
2639                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2640                 MRVL_LOG(INFO, "Parsing config file!");
2641                 if (cfgnum > 1) {
2642                         MRVL_LOG(ERR, "Cannot handle more than one config file!");
2643                         goto out_free_kvlist;
2644                 } else if (cfgnum == 1) {
2645                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2646                                            mrvl_get_qoscfg, &mrvl_qos_cfg);
2647                 }
2648         }
2649
2650         if (mrvl_dev_num)
2651                 goto init_devices;
2652
2653         MRVL_LOG(INFO, "Perform MUSDK initializations");
2654
2655         ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist);
2656         if (ret)
2657                 goto out_free_kvlist;
2658
2659         ret = mrvl_init_pp2();
2660         if (ret) {
2661                 MRVL_LOG(ERR, "Failed to init PP!");
2662                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2663                 goto out_free_kvlist;
2664         }
2665
2666         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2667         memset(mrvl_port_to_bpool_lookup, 0, sizeof(mrvl_port_to_bpool_lookup));
2668
2669         mrvl_lcore_first = RTE_MAX_LCORE;
2670         mrvl_lcore_last = 0;
2671
2672 init_devices:
2673         for (i = 0; i < ifnum; i++) {
2674                 MRVL_LOG(INFO, "Creating %s", ifnames.names[i]);
2675                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
2676                 if (ret)
2677                         goto out_cleanup;
2678         }
2679         mrvl_dev_num += ifnum;
2680
2681         rte_kvargs_free(kvlist);
2682
2683         return 0;
2684 out_cleanup:
2685         for (; i > 0; i--)
2686                 mrvl_eth_dev_destroy(ifnames.names[i]);
2687
2688         if (mrvl_dev_num == 0) {
2689                 mrvl_deinit_pp2();
2690                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2691         }
2692 out_free_kvlist:
2693         rte_kvargs_free(kvlist);
2694
2695         return ret;
2696 }
2697
2698 /**
2699  * DPDK callback to remove virtual device.
2700  *
2701  * @param vdev
2702  *   Pointer to the removed virtual device.
2703  *
2704  * @return
2705  *   0 on success, negative error value otherwise.
2706  */
2707 static int
2708 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2709 {
2710         int i;
2711         const char *name;
2712
2713         name = rte_vdev_device_name(vdev);
2714         if (!name)
2715                 return -EINVAL;
2716
2717         MRVL_LOG(INFO, "Removing %s", name);
2718
2719         RTE_ETH_FOREACH_DEV(i) { /* FIXME: removing all devices! */
2720                 char ifname[RTE_ETH_NAME_MAX_LEN];
2721
2722                 rte_eth_dev_get_name_by_port(i, ifname);
2723                 mrvl_eth_dev_destroy(ifname);
2724                 mrvl_dev_num--;
2725         }
2726
2727         if (mrvl_dev_num == 0) {
2728                 MRVL_LOG(INFO, "Perform MUSDK deinit");
2729                 mrvl_deinit_hifs();
2730                 mrvl_deinit_pp2();
2731                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2732         }
2733
2734         return 0;
2735 }
2736
2737 static struct rte_vdev_driver pmd_mrvl_drv = {
2738         .probe = rte_pmd_mrvl_probe,
2739         .remove = rte_pmd_mrvl_remove,
2740 };
2741
2742 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv);
2743 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2);
2744
2745 RTE_INIT(mrvl_init_log)
2746 {
2747         mrvl_logtype = rte_log_register("pmd.net.mvpp2");
2748         if (mrvl_logtype >= 0)
2749                 rte_log_set_level(mrvl_logtype, RTE_LOG_NOTICE);
2750 }