net/mvpp2: align with MUSDK 18.09
[dpdk.git] / drivers / net / mvpp2 / mrvl_ethdev.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017 Marvell International Ltd.
3  * Copyright(c) 2017 Semihalf.
4  * All rights reserved.
5  */
6
7 #include <rte_ethdev_driver.h>
8 #include <rte_kvargs.h>
9 #include <rte_log.h>
10 #include <rte_malloc.h>
11 #include <rte_bus_vdev.h>
12
13 #include <fcntl.h>
14 #include <linux/ethtool.h>
15 #include <linux/sockios.h>
16 #include <net/if.h>
17 #include <net/if_arp.h>
18 #include <sys/ioctl.h>
19 #include <sys/socket.h>
20 #include <sys/stat.h>
21 #include <sys/types.h>
22
23 #include <rte_mvep_common.h>
24 #include "mrvl_ethdev.h"
25 #include "mrvl_qos.h"
26 #include "mrvl_flow.h"
27 #include "mrvl_mtr.h"
28 #include "mrvl_tm.h"
29
30 /* bitmask with reserved hifs */
31 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
32 /* bitmask with reserved bpools */
33 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
34 /* bitmask with reserved kernel RSS tables */
35 #define MRVL_MUSDK_RSS_RESERVED 0x01
36 /* maximum number of available hifs */
37 #define MRVL_MUSDK_HIFS_MAX 9
38
39 /* prefetch shift */
40 #define MRVL_MUSDK_PREFETCH_SHIFT 2
41
42 /* TCAM has 25 entries reserved for uc/mc filter entries */
43 #define MRVL_MAC_ADDRS_MAX 25
44 #define MRVL_MATCH_LEN 16
45 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
46 /* Maximum allowable packet size */
47 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
48
49 #define MRVL_IFACE_NAME_ARG "iface"
50 #define MRVL_CFG_ARG "cfg"
51
52 #define MRVL_BURST_SIZE 64
53
54 #define MRVL_ARP_LENGTH 28
55
56 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
57 #define MRVL_COOKIE_HIGH_ADDR_MASK 0xffffff0000000000
58
59 /** Port Rx offload capabilities */
60 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
61                           DEV_RX_OFFLOAD_JUMBO_FRAME | \
62                           DEV_RX_OFFLOAD_CHECKSUM)
63
64 /** Port Tx offloads capabilities */
65 #define MRVL_TX_OFFLOADS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
66                           DEV_TX_OFFLOAD_UDP_CKSUM | \
67                           DEV_TX_OFFLOAD_TCP_CKSUM)
68
69 static const char * const valid_args[] = {
70         MRVL_IFACE_NAME_ARG,
71         MRVL_CFG_ARG,
72         NULL
73 };
74
75 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
76 static struct pp2_hif *hifs[RTE_MAX_LCORE];
77 static int used_bpools[PP2_NUM_PKT_PROC] = {
78         [0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED
79 };
80
81 static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
82 static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
83 static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
84
85 int mrvl_logtype;
86
87 struct mrvl_ifnames {
88         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
89         int idx;
90 };
91
92 /*
93  * To use buffer harvesting based on loopback port shadow queue structure
94  * was introduced for buffers information bookkeeping.
95  *
96  * Before sending the packet, related buffer information (pp2_buff_inf) is
97  * stored in shadow queue. After packet is transmitted no longer used
98  * packet buffer is released back to it's original hardware pool,
99  * on condition it originated from interface.
100  * In case it  was generated by application itself i.e: mbuf->port field is
101  * 0xff then its released to software mempool.
102  */
103 struct mrvl_shadow_txq {
104         int head;           /* write index - used when sending buffers */
105         int tail;           /* read index - used when releasing buffers */
106         u16 size;           /* queue occupied size */
107         u16 num_to_release; /* number of buffers sent, that can be released */
108         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
109 };
110
111 struct mrvl_rxq {
112         struct mrvl_priv *priv;
113         struct rte_mempool *mp;
114         int queue_id;
115         int port_id;
116         int cksum_enabled;
117         uint64_t bytes_recv;
118         uint64_t drop_mac;
119 };
120
121 struct mrvl_txq {
122         struct mrvl_priv *priv;
123         int queue_id;
124         int port_id;
125         uint64_t bytes_sent;
126         struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE];
127         int tx_deferred_start;
128 };
129
130 static int mrvl_lcore_first;
131 static int mrvl_lcore_last;
132 static int mrvl_dev_num;
133
134 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
135 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
136                         struct pp2_hif *hif, unsigned int core_id,
137                         struct mrvl_shadow_txq *sq, int qid, int force);
138
139 #define MRVL_XSTATS_TBL_ENTRY(name) { \
140         #name, offsetof(struct pp2_ppio_statistics, name),      \
141         sizeof(((struct pp2_ppio_statistics *)0)->name)         \
142 }
143
144 /* Table with xstats data */
145 static struct {
146         const char *name;
147         unsigned int offset;
148         unsigned int size;
149 } mrvl_xstats_tbl[] = {
150         MRVL_XSTATS_TBL_ENTRY(rx_bytes),
151         MRVL_XSTATS_TBL_ENTRY(rx_packets),
152         MRVL_XSTATS_TBL_ENTRY(rx_unicast_packets),
153         MRVL_XSTATS_TBL_ENTRY(rx_errors),
154         MRVL_XSTATS_TBL_ENTRY(rx_fullq_dropped),
155         MRVL_XSTATS_TBL_ENTRY(rx_bm_dropped),
156         MRVL_XSTATS_TBL_ENTRY(rx_early_dropped),
157         MRVL_XSTATS_TBL_ENTRY(rx_fifo_dropped),
158         MRVL_XSTATS_TBL_ENTRY(rx_cls_dropped),
159         MRVL_XSTATS_TBL_ENTRY(tx_bytes),
160         MRVL_XSTATS_TBL_ENTRY(tx_packets),
161         MRVL_XSTATS_TBL_ENTRY(tx_unicast_packets),
162         MRVL_XSTATS_TBL_ENTRY(tx_errors)
163 };
164
165 static inline int
166 mrvl_get_bpool_size(int pp2_id, int pool_id)
167 {
168         int i;
169         int size = 0;
170
171         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
172                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
173
174         return size;
175 }
176
177 static inline int
178 mrvl_reserve_bit(int *bitmap, int max)
179 {
180         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
181
182         if (n >= max)
183                 return -1;
184
185         *bitmap |= 1 << n;
186
187         return n;
188 }
189
190 static int
191 mrvl_init_hif(int core_id)
192 {
193         struct pp2_hif_params params;
194         char match[MRVL_MATCH_LEN];
195         int ret;
196
197         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
198         if (ret < 0) {
199                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
200                 return ret;
201         }
202
203         snprintf(match, sizeof(match), "hif-%d", ret);
204         memset(&params, 0, sizeof(params));
205         params.match = match;
206         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
207         ret = pp2_hif_init(&params, &hifs[core_id]);
208         if (ret) {
209                 MRVL_LOG(ERR, "Failed to initialize hif %d", core_id);
210                 return ret;
211         }
212
213         return 0;
214 }
215
216 static inline struct pp2_hif*
217 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
218 {
219         int ret;
220
221         if (likely(hifs[core_id] != NULL))
222                 return hifs[core_id];
223
224         rte_spinlock_lock(&priv->lock);
225
226         ret = mrvl_init_hif(core_id);
227         if (ret < 0) {
228                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
229                 goto out;
230         }
231
232         if (core_id < mrvl_lcore_first)
233                 mrvl_lcore_first = core_id;
234
235         if (core_id > mrvl_lcore_last)
236                 mrvl_lcore_last = core_id;
237 out:
238         rte_spinlock_unlock(&priv->lock);
239
240         return hifs[core_id];
241 }
242
243 /**
244  * Configure rss based on dpdk rss configuration.
245  *
246  * @param priv
247  *   Pointer to private structure.
248  * @param rss_conf
249  *   Pointer to RSS configuration.
250  *
251  * @return
252  *   0 on success, negative error value otherwise.
253  */
254 static int
255 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
256 {
257         if (rss_conf->rss_key)
258                 MRVL_LOG(WARNING, "Changing hash key is not supported");
259
260         if (rss_conf->rss_hf == 0) {
261                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
262         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
263                 priv->ppio_params.inqs_params.hash_type =
264                         PP2_PPIO_HASH_T_2_TUPLE;
265         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
266                 priv->ppio_params.inqs_params.hash_type =
267                         PP2_PPIO_HASH_T_5_TUPLE;
268                 priv->rss_hf_tcp = 1;
269         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
270                 priv->ppio_params.inqs_params.hash_type =
271                         PP2_PPIO_HASH_T_5_TUPLE;
272                 priv->rss_hf_tcp = 0;
273         } else {
274                 return -EINVAL;
275         }
276
277         return 0;
278 }
279
280 /**
281  * Ethernet device configuration.
282  *
283  * Prepare the driver for a given number of TX and RX queues and
284  * configure RSS.
285  *
286  * @param dev
287  *   Pointer to Ethernet device structure.
288  *
289  * @return
290  *   0 on success, negative error value otherwise.
291  */
292 static int
293 mrvl_dev_configure(struct rte_eth_dev *dev)
294 {
295         struct mrvl_priv *priv = dev->data->dev_private;
296         int ret;
297
298         if (priv->ppio) {
299                 MRVL_LOG(INFO, "Device reconfiguration is not supported");
300                 return -EINVAL;
301         }
302
303         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
304             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
305                 MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
306                         dev->data->dev_conf.rxmode.mq_mode);
307                 return -EINVAL;
308         }
309
310         if (dev->data->dev_conf.rxmode.split_hdr_size) {
311                 MRVL_LOG(INFO, "Split headers not supported");
312                 return -EINVAL;
313         }
314
315         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
316                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
317                                  MRVL_PP2_ETH_HDRS_LEN;
318
319         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
320                                   dev->data->nb_rx_queues);
321         if (ret < 0)
322                 return ret;
323
324         ret = mrvl_configure_txqs(priv, dev->data->port_id,
325                                   dev->data->nb_tx_queues);
326         if (ret < 0)
327                 return ret;
328
329         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
330         priv->ppio_params.maintain_stats = 1;
331         priv->nb_rx_queues = dev->data->nb_rx_queues;
332
333         ret = mrvl_tm_init(dev);
334         if (ret < 0)
335                 return ret;
336
337         if (dev->data->nb_rx_queues == 1 &&
338             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
339                 MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
340                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
341
342                 return 0;
343         }
344
345         return mrvl_configure_rss(priv,
346                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
347 }
348
349 /**
350  * DPDK callback to change the MTU.
351  *
352  * Setting the MTU affects hardware MRU (packets larger than the MRU
353  * will be dropped).
354  *
355  * @param dev
356  *   Pointer to Ethernet device structure.
357  * @param mtu
358  *   New MTU.
359  *
360  * @return
361  *   0 on success, negative error value otherwise.
362  */
363 static int
364 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
365 {
366         struct mrvl_priv *priv = dev->data->dev_private;
367         uint16_t mru;
368         uint16_t mbuf_data_size = 0; /* SW buffer size */
369         int ret;
370
371         mru = MRVL_PP2_MTU_TO_MRU(mtu);
372         /*
373          * min_rx_buf_size is equal to mbuf data size
374          * if pmd didn't set it differently
375          */
376         mbuf_data_size = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
377         /* Prevent PMD from:
378          * - setting mru greater than the mbuf size resulting in
379          * hw and sw buffer size mismatch
380          * - setting mtu that requires the support of scattered packets
381          * when this feature has not been enabled/supported so far
382          * (TODO check scattered_rx flag here once scattered RX is supported).
383          */
384         if (mru + MRVL_PKT_OFFS > mbuf_data_size) {
385                 mru = mbuf_data_size - MRVL_PKT_OFFS;
386                 mtu = MRVL_PP2_MRU_TO_MTU(mru);
387                 MRVL_LOG(WARNING, "MTU too big, max MTU possible limitted "
388                         "by current mbuf size: %u. Set MTU to %u, MRU to %u",
389                         mbuf_data_size, mtu, mru);
390         }
391
392         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX) {
393                 MRVL_LOG(ERR, "Invalid MTU [%u] or MRU [%u]", mtu, mru);
394                 return -EINVAL;
395         }
396
397         dev->data->mtu = mtu;
398         dev->data->dev_conf.rxmode.max_rx_pkt_len = mru - MV_MH_SIZE;
399
400         if (!priv->ppio)
401                 return 0;
402
403         ret = pp2_ppio_set_mru(priv->ppio, mru);
404         if (ret) {
405                 MRVL_LOG(ERR, "Failed to change MRU");
406                 return ret;
407         }
408
409         ret = pp2_ppio_set_mtu(priv->ppio, mtu);
410         if (ret) {
411                 MRVL_LOG(ERR, "Failed to change MTU");
412                 return ret;
413         }
414
415         return 0;
416 }
417
418 /**
419  * DPDK callback to bring the link up.
420  *
421  * @param dev
422  *   Pointer to Ethernet device structure.
423  *
424  * @return
425  *   0 on success, negative error value otherwise.
426  */
427 static int
428 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
429 {
430         struct mrvl_priv *priv = dev->data->dev_private;
431         int ret;
432
433         if (!priv->ppio)
434                 return -EPERM;
435
436         ret = pp2_ppio_enable(priv->ppio);
437         if (ret)
438                 return ret;
439
440         /*
441          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
442          * as pp2_ppio_enable() changes port->t_mode from default 0 to
443          * PP2_TRAFFIC_INGRESS_EGRESS.
444          *
445          * Set mtu to default DPDK value here.
446          */
447         ret = mrvl_mtu_set(dev, dev->data->mtu);
448         if (ret)
449                 pp2_ppio_disable(priv->ppio);
450
451         return ret;
452 }
453
454 /**
455  * DPDK callback to bring the link down.
456  *
457  * @param dev
458  *   Pointer to Ethernet device structure.
459  *
460  * @return
461  *   0 on success, negative error value otherwise.
462  */
463 static int
464 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
465 {
466         struct mrvl_priv *priv = dev->data->dev_private;
467
468         if (!priv->ppio)
469                 return -EPERM;
470
471         return pp2_ppio_disable(priv->ppio);
472 }
473
474 /**
475  * DPDK callback to start tx queue.
476  *
477  * @param dev
478  *   Pointer to Ethernet device structure.
479  * @param queue_id
480  *   Transmit queue index.
481  *
482  * @return
483  *   0 on success, negative error value otherwise.
484  */
485 static int
486 mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
487 {
488         struct mrvl_priv *priv = dev->data->dev_private;
489         int ret;
490
491         if (!priv)
492                 return -EPERM;
493
494         /* passing 1 enables given tx queue */
495         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1);
496         if (ret) {
497                 MRVL_LOG(ERR, "Failed to start txq %d", queue_id);
498                 return ret;
499         }
500
501         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
502
503         return 0;
504 }
505
506 /**
507  * DPDK callback to stop tx queue.
508  *
509  * @param dev
510  *   Pointer to Ethernet device structure.
511  * @param queue_id
512  *   Transmit queue index.
513  *
514  * @return
515  *   0 on success, negative error value otherwise.
516  */
517 static int
518 mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
519 {
520         struct mrvl_priv *priv = dev->data->dev_private;
521         int ret;
522
523         if (!priv->ppio)
524                 return -EPERM;
525
526         /* passing 0 disables given tx queue */
527         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0);
528         if (ret) {
529                 MRVL_LOG(ERR, "Failed to stop txq %d", queue_id);
530                 return ret;
531         }
532
533         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
534
535         return 0;
536 }
537
538 /**
539  * DPDK callback to start the device.
540  *
541  * @param dev
542  *   Pointer to Ethernet device structure.
543  *
544  * @return
545  *   0 on success, negative errno value on failure.
546  */
547 static int
548 mrvl_dev_start(struct rte_eth_dev *dev)
549 {
550         struct mrvl_priv *priv = dev->data->dev_private;
551         char match[MRVL_MATCH_LEN];
552         int ret = 0, i, def_init_size;
553
554         if (priv->ppio)
555                 return mrvl_dev_set_link_up(dev);
556
557         snprintf(match, sizeof(match), "ppio-%d:%d",
558                  priv->pp_id, priv->ppio_id);
559         priv->ppio_params.match = match;
560
561         /*
562          * Calculate the minimum bpool size for refill feature as follows:
563          * 2 default burst sizes multiply by number of rx queues.
564          * If the bpool size will be below this value, new buffers will
565          * be added to the pool.
566          */
567         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
568
569         /* In case initial bpool size configured in queues setup is
570          * smaller than minimum size add more buffers
571          */
572         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
573         if (priv->bpool_init_size < def_init_size) {
574                 int buffs_to_add = def_init_size - priv->bpool_init_size;
575
576                 priv->bpool_init_size += buffs_to_add;
577                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
578                 if (ret)
579                         MRVL_LOG(ERR, "Failed to add buffers to bpool");
580         }
581
582         /*
583          * Calculate the maximum bpool size for refill feature as follows:
584          * maximum number of descriptors in rx queue multiply by number
585          * of rx queues plus minimum bpool size.
586          * In case the bpool size will exceed this value, superfluous buffers
587          * will be removed
588          */
589         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
590                                 priv->bpool_min_size;
591
592         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
593         if (ret) {
594                 MRVL_LOG(ERR, "Failed to init ppio");
595                 return ret;
596         }
597
598         /*
599          * In case there are some some stale uc/mc mac addresses flush them
600          * here. It cannot be done during mrvl_dev_close() as port information
601          * is already gone at that point (due to pp2_ppio_deinit() in
602          * mrvl_dev_stop()).
603          */
604         if (!priv->uc_mc_flushed) {
605                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
606                 if (ret) {
607                         MRVL_LOG(ERR,
608                                 "Failed to flush uc/mc filter list");
609                         goto out;
610                 }
611                 priv->uc_mc_flushed = 1;
612         }
613
614         if (!priv->vlan_flushed) {
615                 ret = pp2_ppio_flush_vlan(priv->ppio);
616                 if (ret) {
617                         MRVL_LOG(ERR, "Failed to flush vlan list");
618                         /*
619                          * TODO
620                          * once pp2_ppio_flush_vlan() is supported jump to out
621                          * goto out;
622                          */
623                 }
624                 priv->vlan_flushed = 1;
625         }
626         ret = mrvl_mtu_set(dev, dev->data->mtu);
627         if (ret)
628                 MRVL_LOG(ERR, "Failed to set MTU to %d", dev->data->mtu);
629
630         /* For default QoS config, don't start classifier. */
631         if (mrvl_qos_cfg  &&
632             mrvl_qos_cfg->port[dev->data->port_id].use_global_defaults == 0) {
633                 ret = mrvl_start_qos_mapping(priv);
634                 if (ret) {
635                         MRVL_LOG(ERR, "Failed to setup QoS mapping");
636                         goto out;
637                 }
638         }
639
640         ret = mrvl_dev_set_link_up(dev);
641         if (ret) {
642                 MRVL_LOG(ERR, "Failed to set link up");
643                 goto out;
644         }
645
646         /* start tx queues */
647         for (i = 0; i < dev->data->nb_tx_queues; i++) {
648                 struct mrvl_txq *txq = dev->data->tx_queues[i];
649
650                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
651
652                 if (!txq->tx_deferred_start)
653                         continue;
654
655                 /*
656                  * All txqs are started by default. Stop them
657                  * so that tx_deferred_start works as expected.
658                  */
659                 ret = mrvl_tx_queue_stop(dev, i);
660                 if (ret)
661                         goto out;
662         }
663
664         mrvl_flow_init(dev);
665         mrvl_mtr_init(dev);
666
667         return 0;
668 out:
669         MRVL_LOG(ERR, "Failed to start device");
670         pp2_ppio_deinit(priv->ppio);
671         return ret;
672 }
673
674 /**
675  * Flush receive queues.
676  *
677  * @param dev
678  *   Pointer to Ethernet device structure.
679  */
680 static void
681 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
682 {
683         int i;
684
685         MRVL_LOG(INFO, "Flushing rx queues");
686         for (i = 0; i < dev->data->nb_rx_queues; i++) {
687                 int ret, num;
688
689                 do {
690                         struct mrvl_rxq *q = dev->data->rx_queues[i];
691                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
692
693                         num = MRVL_PP2_RXD_MAX;
694                         ret = pp2_ppio_recv(q->priv->ppio,
695                                             q->priv->rxq_map[q->queue_id].tc,
696                                             q->priv->rxq_map[q->queue_id].inq,
697                                             descs, (uint16_t *)&num);
698                 } while (ret == 0 && num);
699         }
700 }
701
702 /**
703  * Flush transmit shadow queues.
704  *
705  * @param dev
706  *   Pointer to Ethernet device structure.
707  */
708 static void
709 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
710 {
711         int i, j;
712         struct mrvl_txq *txq;
713
714         MRVL_LOG(INFO, "Flushing tx shadow queues");
715         for (i = 0; i < dev->data->nb_tx_queues; i++) {
716                 txq = (struct mrvl_txq *)dev->data->tx_queues[i];
717
718                 for (j = 0; j < RTE_MAX_LCORE; j++) {
719                         struct mrvl_shadow_txq *sq;
720
721                         if (!hifs[j])
722                                 continue;
723
724                         sq = &txq->shadow_txqs[j];
725                         mrvl_free_sent_buffers(txq->priv->ppio,
726                                 hifs[j], j, sq, txq->queue_id, 1);
727                         while (sq->tail != sq->head) {
728                                 uint64_t addr = cookie_addr_high |
729                                         sq->ent[sq->tail].buff.cookie;
730                                 rte_pktmbuf_free(
731                                         (struct rte_mbuf *)addr);
732                                 sq->tail = (sq->tail + 1) &
733                                             MRVL_PP2_TX_SHADOWQ_MASK;
734                         }
735                         memset(sq, 0, sizeof(*sq));
736                 }
737         }
738 }
739
740 /**
741  * Flush hardware bpool (buffer-pool).
742  *
743  * @param dev
744  *   Pointer to Ethernet device structure.
745  */
746 static void
747 mrvl_flush_bpool(struct rte_eth_dev *dev)
748 {
749         struct mrvl_priv *priv = dev->data->dev_private;
750         struct pp2_hif *hif;
751         uint32_t num;
752         int ret;
753         unsigned int core_id = rte_lcore_id();
754
755         if (core_id == LCORE_ID_ANY)
756                 core_id = 0;
757
758         hif = mrvl_get_hif(priv, core_id);
759
760         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
761         if (ret) {
762                 MRVL_LOG(ERR, "Failed to get bpool buffers number");
763                 return;
764         }
765
766         while (num--) {
767                 struct pp2_buff_inf inf;
768                 uint64_t addr;
769
770                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
771                 if (ret)
772                         break;
773
774                 addr = cookie_addr_high | inf.cookie;
775                 rte_pktmbuf_free((struct rte_mbuf *)addr);
776         }
777 }
778
779 /**
780  * DPDK callback to stop the device.
781  *
782  * @param dev
783  *   Pointer to Ethernet device structure.
784  */
785 static void
786 mrvl_dev_stop(struct rte_eth_dev *dev)
787 {
788         mrvl_dev_set_link_down(dev);
789 }
790
791 /**
792  * DPDK callback to close the device.
793  *
794  * @param dev
795  *   Pointer to Ethernet device structure.
796  */
797 static void
798 mrvl_dev_close(struct rte_eth_dev *dev)
799 {
800         struct mrvl_priv *priv = dev->data->dev_private;
801         size_t i;
802
803         mrvl_flush_rx_queues(dev);
804         mrvl_flush_tx_shadow_queues(dev);
805         mrvl_flow_deinit(dev);
806         mrvl_mtr_deinit(dev);
807
808         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
809                 struct pp2_ppio_tc_params *tc_params =
810                         &priv->ppio_params.inqs_params.tcs_params[i];
811
812                 if (tc_params->inqs_params) {
813                         rte_free(tc_params->inqs_params);
814                         tc_params->inqs_params = NULL;
815                 }
816         }
817
818         if (priv->cls_tbl) {
819                 pp2_cls_tbl_deinit(priv->cls_tbl);
820                 priv->cls_tbl = NULL;
821         }
822
823         if (priv->qos_tbl) {
824                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
825                 priv->qos_tbl = NULL;
826         }
827
828         mrvl_flush_bpool(dev);
829         mrvl_tm_deinit(dev);
830
831         if (priv->ppio) {
832                 pp2_ppio_deinit(priv->ppio);
833                 priv->ppio = NULL;
834         }
835
836         /* policer must be released after ppio deinitialization */
837         if (priv->default_policer) {
838                 pp2_cls_plcr_deinit(priv->default_policer);
839                 priv->default_policer = NULL;
840         }
841 }
842
843 /**
844  * DPDK callback to retrieve physical link information.
845  *
846  * @param dev
847  *   Pointer to Ethernet device structure.
848  * @param wait_to_complete
849  *   Wait for request completion (ignored).
850  *
851  * @return
852  *   0 on success, negative error value otherwise.
853  */
854 static int
855 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
856 {
857         /*
858          * TODO
859          * once MUSDK provides necessary API use it here
860          */
861         struct mrvl_priv *priv = dev->data->dev_private;
862         struct ethtool_cmd edata;
863         struct ifreq req;
864         int ret, fd, link_up;
865
866         if (!priv->ppio)
867                 return -EPERM;
868
869         edata.cmd = ETHTOOL_GSET;
870
871         strcpy(req.ifr_name, dev->data->name);
872         req.ifr_data = (void *)&edata;
873
874         fd = socket(AF_INET, SOCK_DGRAM, 0);
875         if (fd == -1)
876                 return -EFAULT;
877
878         ret = ioctl(fd, SIOCETHTOOL, &req);
879         if (ret == -1) {
880                 close(fd);
881                 return -EFAULT;
882         }
883
884         close(fd);
885
886         switch (ethtool_cmd_speed(&edata)) {
887         case SPEED_10:
888                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
889                 break;
890         case SPEED_100:
891                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
892                 break;
893         case SPEED_1000:
894                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
895                 break;
896         case SPEED_10000:
897                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
898                 break;
899         default:
900                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
901         }
902
903         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
904                                                          ETH_LINK_HALF_DUPLEX;
905         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
906                                                            ETH_LINK_FIXED;
907         pp2_ppio_get_link_state(priv->ppio, &link_up);
908         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
909
910         return 0;
911 }
912
913 /**
914  * DPDK callback to enable promiscuous mode.
915  *
916  * @param dev
917  *   Pointer to Ethernet device structure.
918  */
919 static void
920 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
921 {
922         struct mrvl_priv *priv = dev->data->dev_private;
923         int ret;
924
925         if (!priv->ppio)
926                 return;
927
928         if (priv->isolated)
929                 return;
930
931         ret = pp2_ppio_set_promisc(priv->ppio, 1);
932         if (ret)
933                 MRVL_LOG(ERR, "Failed to enable promiscuous mode");
934 }
935
936 /**
937  * DPDK callback to enable allmulti mode.
938  *
939  * @param dev
940  *   Pointer to Ethernet device structure.
941  */
942 static void
943 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
944 {
945         struct mrvl_priv *priv = dev->data->dev_private;
946         int ret;
947
948         if (!priv->ppio)
949                 return;
950
951         if (priv->isolated)
952                 return;
953
954         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
955         if (ret)
956                 MRVL_LOG(ERR, "Failed enable all-multicast mode");
957 }
958
959 /**
960  * DPDK callback to disable promiscuous mode.
961  *
962  * @param dev
963  *   Pointer to Ethernet device structure.
964  */
965 static void
966 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
967 {
968         struct mrvl_priv *priv = dev->data->dev_private;
969         int ret;
970
971         if (!priv->ppio)
972                 return;
973
974         ret = pp2_ppio_set_promisc(priv->ppio, 0);
975         if (ret)
976                 MRVL_LOG(ERR, "Failed to disable promiscuous mode");
977 }
978
979 /**
980  * DPDK callback to disable allmulticast mode.
981  *
982  * @param dev
983  *   Pointer to Ethernet device structure.
984  */
985 static void
986 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
987 {
988         struct mrvl_priv *priv = dev->data->dev_private;
989         int ret;
990
991         if (!priv->ppio)
992                 return;
993
994         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
995         if (ret)
996                 MRVL_LOG(ERR, "Failed to disable all-multicast mode");
997 }
998
999 /**
1000  * DPDK callback to remove a MAC address.
1001  *
1002  * @param dev
1003  *   Pointer to Ethernet device structure.
1004  * @param index
1005  *   MAC address index.
1006  */
1007 static void
1008 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1009 {
1010         struct mrvl_priv *priv = dev->data->dev_private;
1011         char buf[ETHER_ADDR_FMT_SIZE];
1012         int ret;
1013
1014         if (!priv->ppio)
1015                 return;
1016
1017         if (priv->isolated)
1018                 return;
1019
1020         ret = pp2_ppio_remove_mac_addr(priv->ppio,
1021                                        dev->data->mac_addrs[index].addr_bytes);
1022         if (ret) {
1023                 ether_format_addr(buf, sizeof(buf),
1024                                   &dev->data->mac_addrs[index]);
1025                 MRVL_LOG(ERR, "Failed to remove mac %s", buf);
1026         }
1027 }
1028
1029 /**
1030  * DPDK callback to add a MAC address.
1031  *
1032  * @param dev
1033  *   Pointer to Ethernet device structure.
1034  * @param mac_addr
1035  *   MAC address to register.
1036  * @param index
1037  *   MAC address index.
1038  * @param vmdq
1039  *   VMDq pool index to associate address with (unused).
1040  *
1041  * @return
1042  *   0 on success, negative error value otherwise.
1043  */
1044 static int
1045 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
1046                   uint32_t index, uint32_t vmdq __rte_unused)
1047 {
1048         struct mrvl_priv *priv = dev->data->dev_private;
1049         char buf[ETHER_ADDR_FMT_SIZE];
1050         int ret;
1051
1052         if (priv->isolated)
1053                 return -ENOTSUP;
1054
1055         if (index == 0)
1056                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
1057                 return -1;
1058
1059         if (!priv->ppio)
1060                 return 0;
1061
1062         /*
1063          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
1064          * parameter uc_filter_max. Maximum number of mc addresses is then
1065          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
1066          * 21 respectively.
1067          *
1068          * If more than uc_filter_max uc addresses were added to filter list
1069          * then NIC will switch to promiscuous mode automatically.
1070          *
1071          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
1072          * were added to filter list then NIC will switch to all-multicast mode
1073          * automatically.
1074          */
1075         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
1076         if (ret) {
1077                 ether_format_addr(buf, sizeof(buf), mac_addr);
1078                 MRVL_LOG(ERR, "Failed to add mac %s", buf);
1079                 return -1;
1080         }
1081
1082         return 0;
1083 }
1084
1085 /**
1086  * DPDK callback to set the primary MAC address.
1087  *
1088  * @param dev
1089  *   Pointer to Ethernet device structure.
1090  * @param mac_addr
1091  *   MAC address to register.
1092  *
1093  * @return
1094  *   0 on success, negative error value otherwise.
1095  */
1096 static int
1097 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1098 {
1099         struct mrvl_priv *priv = dev->data->dev_private;
1100         int ret;
1101
1102         if (!priv->ppio)
1103                 return 0;
1104
1105         if (priv->isolated)
1106                 return -ENOTSUP;
1107
1108         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
1109         if (ret) {
1110                 char buf[ETHER_ADDR_FMT_SIZE];
1111                 ether_format_addr(buf, sizeof(buf), mac_addr);
1112                 MRVL_LOG(ERR, "Failed to set mac to %s", buf);
1113         }
1114
1115         return ret;
1116 }
1117
1118 /**
1119  * DPDK callback to get device statistics.
1120  *
1121  * @param dev
1122  *   Pointer to Ethernet device structure.
1123  * @param stats
1124  *   Stats structure output buffer.
1125  *
1126  * @return
1127  *   0 on success, negative error value otherwise.
1128  */
1129 static int
1130 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1131 {
1132         struct mrvl_priv *priv = dev->data->dev_private;
1133         struct pp2_ppio_statistics ppio_stats;
1134         uint64_t drop_mac = 0;
1135         unsigned int i, idx, ret;
1136
1137         if (!priv->ppio)
1138                 return -EPERM;
1139
1140         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1141                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1142                 struct pp2_ppio_inq_statistics rx_stats;
1143
1144                 if (!rxq)
1145                         continue;
1146
1147                 idx = rxq->queue_id;
1148                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1149                         MRVL_LOG(ERR,
1150                                 "rx queue %d stats out of range (0 - %d)",
1151                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1152                         continue;
1153                 }
1154
1155                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
1156                                                   priv->rxq_map[idx].tc,
1157                                                   priv->rxq_map[idx].inq,
1158                                                   &rx_stats, 0);
1159                 if (unlikely(ret)) {
1160                         MRVL_LOG(ERR,
1161                                 "Failed to update rx queue %d stats", idx);
1162                         break;
1163                 }
1164
1165                 stats->q_ibytes[idx] = rxq->bytes_recv;
1166                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
1167                 stats->q_errors[idx] = rx_stats.drop_early +
1168                                        rx_stats.drop_fullq +
1169                                        rx_stats.drop_bm +
1170                                        rxq->drop_mac;
1171                 stats->ibytes += rxq->bytes_recv;
1172                 drop_mac += rxq->drop_mac;
1173         }
1174
1175         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1176                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1177                 struct pp2_ppio_outq_statistics tx_stats;
1178
1179                 if (!txq)
1180                         continue;
1181
1182                 idx = txq->queue_id;
1183                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1184                         MRVL_LOG(ERR,
1185                                 "tx queue %d stats out of range (0 - %d)",
1186                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1187                 }
1188
1189                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1190                                                    &tx_stats, 0);
1191                 if (unlikely(ret)) {
1192                         MRVL_LOG(ERR,
1193                                 "Failed to update tx queue %d stats", idx);
1194                         break;
1195                 }
1196
1197                 stats->q_opackets[idx] = tx_stats.deq_desc;
1198                 stats->q_obytes[idx] = txq->bytes_sent;
1199                 stats->obytes += txq->bytes_sent;
1200         }
1201
1202         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1203         if (unlikely(ret)) {
1204                 MRVL_LOG(ERR, "Failed to update port statistics");
1205                 return ret;
1206         }
1207
1208         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1209         stats->opackets += ppio_stats.tx_packets;
1210         stats->imissed += ppio_stats.rx_fullq_dropped +
1211                           ppio_stats.rx_bm_dropped +
1212                           ppio_stats.rx_early_dropped +
1213                           ppio_stats.rx_fifo_dropped +
1214                           ppio_stats.rx_cls_dropped;
1215         stats->ierrors = drop_mac;
1216
1217         return 0;
1218 }
1219
1220 /**
1221  * DPDK callback to clear device statistics.
1222  *
1223  * @param dev
1224  *   Pointer to Ethernet device structure.
1225  */
1226 static void
1227 mrvl_stats_reset(struct rte_eth_dev *dev)
1228 {
1229         struct mrvl_priv *priv = dev->data->dev_private;
1230         int i;
1231
1232         if (!priv->ppio)
1233                 return;
1234
1235         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1236                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1237
1238                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1239                                             priv->rxq_map[i].inq, NULL, 1);
1240                 rxq->bytes_recv = 0;
1241                 rxq->drop_mac = 0;
1242         }
1243
1244         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1245                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1246
1247                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1248                 txq->bytes_sent = 0;
1249         }
1250
1251         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1252 }
1253
1254 /**
1255  * DPDK callback to get extended statistics.
1256  *
1257  * @param dev
1258  *   Pointer to Ethernet device structure.
1259  * @param stats
1260  *   Pointer to xstats table.
1261  * @param n
1262  *   Number of entries in xstats table.
1263  * @return
1264  *   Negative value on error, number of read xstats otherwise.
1265  */
1266 static int
1267 mrvl_xstats_get(struct rte_eth_dev *dev,
1268                 struct rte_eth_xstat *stats, unsigned int n)
1269 {
1270         struct mrvl_priv *priv = dev->data->dev_private;
1271         struct pp2_ppio_statistics ppio_stats;
1272         unsigned int i;
1273
1274         if (!stats)
1275                 return 0;
1276
1277         pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1278         for (i = 0; i < n && i < RTE_DIM(mrvl_xstats_tbl); i++) {
1279                 uint64_t val;
1280
1281                 if (mrvl_xstats_tbl[i].size == sizeof(uint32_t))
1282                         val = *(uint32_t *)((uint8_t *)&ppio_stats +
1283                                             mrvl_xstats_tbl[i].offset);
1284                 else if (mrvl_xstats_tbl[i].size == sizeof(uint64_t))
1285                         val = *(uint64_t *)((uint8_t *)&ppio_stats +
1286                                             mrvl_xstats_tbl[i].offset);
1287                 else
1288                         return -EINVAL;
1289
1290                 stats[i].id = i;
1291                 stats[i].value = val;
1292         }
1293
1294         return n;
1295 }
1296
1297 /**
1298  * DPDK callback to reset extended statistics.
1299  *
1300  * @param dev
1301  *   Pointer to Ethernet device structure.
1302  */
1303 static void
1304 mrvl_xstats_reset(struct rte_eth_dev *dev)
1305 {
1306         mrvl_stats_reset(dev);
1307 }
1308
1309 /**
1310  * DPDK callback to get extended statistics names.
1311  *
1312  * @param dev (unused)
1313  *   Pointer to Ethernet device structure.
1314  * @param xstats_names
1315  *   Pointer to xstats names table.
1316  * @param size
1317  *   Size of the xstats names table.
1318  * @return
1319  *   Number of read names.
1320  */
1321 static int
1322 mrvl_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
1323                       struct rte_eth_xstat_name *xstats_names,
1324                       unsigned int size)
1325 {
1326         unsigned int i;
1327
1328         if (!xstats_names)
1329                 return RTE_DIM(mrvl_xstats_tbl);
1330
1331         for (i = 0; i < size && i < RTE_DIM(mrvl_xstats_tbl); i++)
1332                 snprintf(xstats_names[i].name, RTE_ETH_XSTATS_NAME_SIZE, "%s",
1333                          mrvl_xstats_tbl[i].name);
1334
1335         return size;
1336 }
1337
1338 /**
1339  * DPDK callback to get information about the device.
1340  *
1341  * @param dev
1342  *   Pointer to Ethernet device structure (unused).
1343  * @param info
1344  *   Info structure output buffer.
1345  */
1346 static void
1347 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
1348                    struct rte_eth_dev_info *info)
1349 {
1350         info->speed_capa = ETH_LINK_SPEED_10M |
1351                            ETH_LINK_SPEED_100M |
1352                            ETH_LINK_SPEED_1G |
1353                            ETH_LINK_SPEED_10G;
1354
1355         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1356         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1357         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1358
1359         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1360         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1361         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1362
1363         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1364         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1365         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1366
1367         info->rx_offload_capa = MRVL_RX_OFFLOADS;
1368         info->rx_queue_offload_capa = MRVL_RX_OFFLOADS;
1369
1370         info->tx_offload_capa = MRVL_TX_OFFLOADS;
1371         info->tx_queue_offload_capa = MRVL_TX_OFFLOADS;
1372
1373         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1374                                        ETH_RSS_NONFRAG_IPV4_TCP |
1375                                        ETH_RSS_NONFRAG_IPV4_UDP;
1376
1377         /* By default packets are dropped if no descriptors are available */
1378         info->default_rxconf.rx_drop_en = 1;
1379
1380         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1381 }
1382
1383 /**
1384  * Return supported packet types.
1385  *
1386  * @param dev
1387  *   Pointer to Ethernet device structure (unused).
1388  *
1389  * @return
1390  *   Const pointer to the table with supported packet types.
1391  */
1392 static const uint32_t *
1393 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1394 {
1395         static const uint32_t ptypes[] = {
1396                 RTE_PTYPE_L2_ETHER,
1397                 RTE_PTYPE_L2_ETHER_VLAN,
1398                 RTE_PTYPE_L2_ETHER_QINQ,
1399                 RTE_PTYPE_L3_IPV4,
1400                 RTE_PTYPE_L3_IPV4_EXT,
1401                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1402                 RTE_PTYPE_L3_IPV6,
1403                 RTE_PTYPE_L3_IPV6_EXT,
1404                 RTE_PTYPE_L2_ETHER_ARP,
1405                 RTE_PTYPE_L4_TCP,
1406                 RTE_PTYPE_L4_UDP
1407         };
1408
1409         return ptypes;
1410 }
1411
1412 /**
1413  * DPDK callback to get information about specific receive queue.
1414  *
1415  * @param dev
1416  *   Pointer to Ethernet device structure.
1417  * @param rx_queue_id
1418  *   Receive queue index.
1419  * @param qinfo
1420  *   Receive queue information structure.
1421  */
1422 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1423                               struct rte_eth_rxq_info *qinfo)
1424 {
1425         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1426         struct mrvl_priv *priv = dev->data->dev_private;
1427         int inq = priv->rxq_map[rx_queue_id].inq;
1428         int tc = priv->rxq_map[rx_queue_id].tc;
1429         struct pp2_ppio_tc_params *tc_params =
1430                 &priv->ppio_params.inqs_params.tcs_params[tc];
1431
1432         qinfo->mp = q->mp;
1433         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1434 }
1435
1436 /**
1437  * DPDK callback to get information about specific transmit queue.
1438  *
1439  * @param dev
1440  *   Pointer to Ethernet device structure.
1441  * @param tx_queue_id
1442  *   Transmit queue index.
1443  * @param qinfo
1444  *   Transmit queue information structure.
1445  */
1446 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1447                               struct rte_eth_txq_info *qinfo)
1448 {
1449         struct mrvl_priv *priv = dev->data->dev_private;
1450         struct mrvl_txq *txq = dev->data->tx_queues[tx_queue_id];
1451
1452         qinfo->nb_desc =
1453                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1454         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1455 }
1456
1457 /**
1458  * DPDK callback to Configure a VLAN filter.
1459  *
1460  * @param dev
1461  *   Pointer to Ethernet device structure.
1462  * @param vlan_id
1463  *   VLAN ID to filter.
1464  * @param on
1465  *   Toggle filter.
1466  *
1467  * @return
1468  *   0 on success, negative error value otherwise.
1469  */
1470 static int
1471 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1472 {
1473         struct mrvl_priv *priv = dev->data->dev_private;
1474
1475         if (!priv->ppio)
1476                 return -EPERM;
1477
1478         if (priv->isolated)
1479                 return -ENOTSUP;
1480
1481         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1482                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1483 }
1484
1485 /**
1486  * Release buffers to hardware bpool (buffer-pool)
1487  *
1488  * @param rxq
1489  *   Receive queue pointer.
1490  * @param num
1491  *   Number of buffers to release to bpool.
1492  *
1493  * @return
1494  *   0 on success, negative error value otherwise.
1495  */
1496 static int
1497 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1498 {
1499         struct buff_release_entry entries[MRVL_PP2_RXD_MAX];
1500         struct rte_mbuf *mbufs[MRVL_PP2_RXD_MAX];
1501         int i, ret;
1502         unsigned int core_id;
1503         struct pp2_hif *hif;
1504         struct pp2_bpool *bpool;
1505
1506         core_id = rte_lcore_id();
1507         if (core_id == LCORE_ID_ANY)
1508                 core_id = 0;
1509
1510         hif = mrvl_get_hif(rxq->priv, core_id);
1511         if (!hif)
1512                 return -1;
1513
1514         bpool = rxq->priv->bpool;
1515
1516         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1517         if (ret)
1518                 return ret;
1519
1520         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1521                 cookie_addr_high =
1522                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1523
1524         for (i = 0; i < num; i++) {
1525                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1526                         != cookie_addr_high) {
1527                         MRVL_LOG(ERR,
1528                                 "mbuf virtual addr high 0x%lx out of range",
1529                                 (uint64_t)mbufs[i] >> 32);
1530                         goto out;
1531                 }
1532
1533                 entries[i].buff.addr =
1534                         rte_mbuf_data_iova_default(mbufs[i]);
1535                 entries[i].buff.cookie = (uint64_t)mbufs[i];
1536                 entries[i].bpool = bpool;
1537         }
1538
1539         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1540         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1541
1542         if (i != num)
1543                 goto out;
1544
1545         return 0;
1546 out:
1547         for (; i < num; i++)
1548                 rte_pktmbuf_free(mbufs[i]);
1549
1550         return -1;
1551 }
1552
1553 /**
1554  * DPDK callback to configure the receive queue.
1555  *
1556  * @param dev
1557  *   Pointer to Ethernet device structure.
1558  * @param idx
1559  *   RX queue index.
1560  * @param desc
1561  *   Number of descriptors to configure in queue.
1562  * @param socket
1563  *   NUMA socket on which memory must be allocated.
1564  * @param conf
1565  *   Thresholds parameters.
1566  * @param mp
1567  *   Memory pool for buffer allocations.
1568  *
1569  * @return
1570  *   0 on success, negative error value otherwise.
1571  */
1572 static int
1573 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1574                     unsigned int socket,
1575                     const struct rte_eth_rxconf *conf,
1576                     struct rte_mempool *mp)
1577 {
1578         struct mrvl_priv *priv = dev->data->dev_private;
1579         struct mrvl_rxq *rxq;
1580         uint32_t frame_size, buf_size = rte_pktmbuf_data_room_size(mp);
1581         uint32_t max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1582         int ret, tc, inq;
1583         uint64_t offloads;
1584
1585         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1586
1587         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1588                 /*
1589                  * Unknown TC mapping, mapping will not have a correct queue.
1590                  */
1591                 MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu",
1592                         idx, priv->ppio_id);
1593                 return -EFAULT;
1594         }
1595
1596         frame_size = buf_size - RTE_PKTMBUF_HEADROOM - MRVL_PKT_EFFEC_OFFS;
1597         if (frame_size < max_rx_pkt_len) {
1598                 MRVL_LOG(WARNING,
1599                         "Mbuf size must be increased to %u bytes to hold up "
1600                         "to %u bytes of data.",
1601                         buf_size + max_rx_pkt_len - frame_size,
1602                         max_rx_pkt_len);
1603                 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
1604                 MRVL_LOG(INFO, "Setting max rx pkt len to %u",
1605                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
1606         }
1607
1608         if (dev->data->rx_queues[idx]) {
1609                 rte_free(dev->data->rx_queues[idx]);
1610                 dev->data->rx_queues[idx] = NULL;
1611         }
1612
1613         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1614         if (!rxq)
1615                 return -ENOMEM;
1616
1617         rxq->priv = priv;
1618         rxq->mp = mp;
1619         rxq->cksum_enabled = offloads & DEV_RX_OFFLOAD_IPV4_CKSUM;
1620         rxq->queue_id = idx;
1621         rxq->port_id = dev->data->port_id;
1622         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1623
1624         tc = priv->rxq_map[rxq->queue_id].tc,
1625         inq = priv->rxq_map[rxq->queue_id].inq;
1626         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1627                 desc;
1628
1629         ret = mrvl_fill_bpool(rxq, desc);
1630         if (ret) {
1631                 rte_free(rxq);
1632                 return ret;
1633         }
1634
1635         priv->bpool_init_size += desc;
1636
1637         dev->data->rx_queues[idx] = rxq;
1638
1639         return 0;
1640 }
1641
1642 /**
1643  * DPDK callback to release the receive queue.
1644  *
1645  * @param rxq
1646  *   Generic receive queue pointer.
1647  */
1648 static void
1649 mrvl_rx_queue_release(void *rxq)
1650 {
1651         struct mrvl_rxq *q = rxq;
1652         struct pp2_ppio_tc_params *tc_params;
1653         int i, num, tc, inq;
1654         struct pp2_hif *hif;
1655         unsigned int core_id = rte_lcore_id();
1656
1657         if (core_id == LCORE_ID_ANY)
1658                 core_id = 0;
1659
1660         if (!q)
1661                 return;
1662
1663         hif = mrvl_get_hif(q->priv, core_id);
1664
1665         if (!hif)
1666                 return;
1667
1668         tc = q->priv->rxq_map[q->queue_id].tc;
1669         inq = q->priv->rxq_map[q->queue_id].inq;
1670         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1671         num = tc_params->inqs_params[inq].size;
1672         for (i = 0; i < num; i++) {
1673                 struct pp2_buff_inf inf;
1674                 uint64_t addr;
1675
1676                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
1677                 addr = cookie_addr_high | inf.cookie;
1678                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1679         }
1680
1681         rte_free(q);
1682 }
1683
1684 /**
1685  * DPDK callback to configure the transmit queue.
1686  *
1687  * @param dev
1688  *   Pointer to Ethernet device structure.
1689  * @param idx
1690  *   Transmit queue index.
1691  * @param desc
1692  *   Number of descriptors to configure in the queue.
1693  * @param socket
1694  *   NUMA socket on which memory must be allocated.
1695  * @param conf
1696  *   Tx queue configuration parameters.
1697  *
1698  * @return
1699  *   0 on success, negative error value otherwise.
1700  */
1701 static int
1702 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1703                     unsigned int socket,
1704                     const struct rte_eth_txconf *conf)
1705 {
1706         struct mrvl_priv *priv = dev->data->dev_private;
1707         struct mrvl_txq *txq;
1708
1709         if (dev->data->tx_queues[idx]) {
1710                 rte_free(dev->data->tx_queues[idx]);
1711                 dev->data->tx_queues[idx] = NULL;
1712         }
1713
1714         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1715         if (!txq)
1716                 return -ENOMEM;
1717
1718         txq->priv = priv;
1719         txq->queue_id = idx;
1720         txq->port_id = dev->data->port_id;
1721         txq->tx_deferred_start = conf->tx_deferred_start;
1722         dev->data->tx_queues[idx] = txq;
1723
1724         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1725
1726         return 0;
1727 }
1728
1729 /**
1730  * DPDK callback to release the transmit queue.
1731  *
1732  * @param txq
1733  *   Generic transmit queue pointer.
1734  */
1735 static void
1736 mrvl_tx_queue_release(void *txq)
1737 {
1738         struct mrvl_txq *q = txq;
1739
1740         if (!q)
1741                 return;
1742
1743         rte_free(q);
1744 }
1745
1746 /**
1747  * DPDK callback to get flow control configuration.
1748  *
1749  * @param dev
1750  *  Pointer to Ethernet device structure.
1751  * @param fc_conf
1752  *  Pointer to the flow control configuration.
1753  *
1754  * @return
1755  *  0 on success, negative error value otherwise.
1756  */
1757 static int
1758 mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1759 {
1760         struct mrvl_priv *priv = dev->data->dev_private;
1761         int ret, en;
1762
1763         if (!priv)
1764                 return -EPERM;
1765
1766         ret = pp2_ppio_get_rx_pause(priv->ppio, &en);
1767         if (ret) {
1768                 MRVL_LOG(ERR, "Failed to read rx pause state");
1769                 return ret;
1770         }
1771
1772         fc_conf->mode = en ? RTE_FC_RX_PAUSE : RTE_FC_NONE;
1773
1774         return 0;
1775 }
1776
1777 /**
1778  * DPDK callback to set flow control configuration.
1779  *
1780  * @param dev
1781  *  Pointer to Ethernet device structure.
1782  * @param fc_conf
1783  *  Pointer to the flow control configuration.
1784  *
1785  * @return
1786  *  0 on success, negative error value otherwise.
1787  */
1788 static int
1789 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1790 {
1791         struct mrvl_priv *priv = dev->data->dev_private;
1792
1793         if (!priv)
1794                 return -EPERM;
1795
1796         if (fc_conf->high_water ||
1797             fc_conf->low_water ||
1798             fc_conf->pause_time ||
1799             fc_conf->mac_ctrl_frame_fwd ||
1800             fc_conf->autoneg) {
1801                 MRVL_LOG(ERR, "Flowctrl parameter is not supported");
1802
1803                 return -EINVAL;
1804         }
1805
1806         if (fc_conf->mode == RTE_FC_NONE ||
1807             fc_conf->mode == RTE_FC_RX_PAUSE) {
1808                 int ret, en;
1809
1810                 en = fc_conf->mode == RTE_FC_NONE ? 0 : 1;
1811                 ret = pp2_ppio_set_rx_pause(priv->ppio, en);
1812                 if (ret)
1813                         MRVL_LOG(ERR,
1814                                 "Failed to change flowctrl on RX side");
1815
1816                 return ret;
1817         }
1818
1819         return 0;
1820 }
1821
1822 /**
1823  * Update RSS hash configuration
1824  *
1825  * @param dev
1826  *   Pointer to Ethernet device structure.
1827  * @param rss_conf
1828  *   Pointer to RSS configuration.
1829  *
1830  * @return
1831  *   0 on success, negative error value otherwise.
1832  */
1833 static int
1834 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1835                      struct rte_eth_rss_conf *rss_conf)
1836 {
1837         struct mrvl_priv *priv = dev->data->dev_private;
1838
1839         if (priv->isolated)
1840                 return -ENOTSUP;
1841
1842         return mrvl_configure_rss(priv, rss_conf);
1843 }
1844
1845 /**
1846  * DPDK callback to get RSS hash configuration.
1847  *
1848  * @param dev
1849  *   Pointer to Ethernet device structure.
1850  * @rss_conf
1851  *   Pointer to RSS configuration.
1852  *
1853  * @return
1854  *   Always 0.
1855  */
1856 static int
1857 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1858                        struct rte_eth_rss_conf *rss_conf)
1859 {
1860         struct mrvl_priv *priv = dev->data->dev_private;
1861         enum pp2_ppio_hash_type hash_type =
1862                 priv->ppio_params.inqs_params.hash_type;
1863
1864         rss_conf->rss_key = NULL;
1865
1866         if (hash_type == PP2_PPIO_HASH_T_NONE)
1867                 rss_conf->rss_hf = 0;
1868         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1869                 rss_conf->rss_hf = ETH_RSS_IPV4;
1870         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1871                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1872         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1873                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1874
1875         return 0;
1876 }
1877
1878 /**
1879  * DPDK callback to get rte_flow callbacks.
1880  *
1881  * @param dev
1882  *   Pointer to the device structure.
1883  * @param filer_type
1884  *   Flow filter type.
1885  * @param filter_op
1886  *   Flow filter operation.
1887  * @param arg
1888  *   Pointer to pass the flow ops.
1889  *
1890  * @return
1891  *   0 on success, negative error value otherwise.
1892  */
1893 static int
1894 mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
1895                      enum rte_filter_type filter_type,
1896                      enum rte_filter_op filter_op, void *arg)
1897 {
1898         switch (filter_type) {
1899         case RTE_ETH_FILTER_GENERIC:
1900                 if (filter_op != RTE_ETH_FILTER_GET)
1901                         return -EINVAL;
1902                 *(const void **)arg = &mrvl_flow_ops;
1903                 return 0;
1904         default:
1905                 MRVL_LOG(WARNING, "Filter type (%d) not supported",
1906                                 filter_type);
1907                 return -EINVAL;
1908         }
1909 }
1910
1911 /**
1912  * DPDK callback to get rte_mtr callbacks.
1913  *
1914  * @param dev
1915  *   Pointer to the device structure.
1916  * @param ops
1917  *   Pointer to pass the mtr ops.
1918  *
1919  * @return
1920  *   Always 0.
1921  */
1922 static int
1923 mrvl_mtr_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
1924 {
1925         *(const void **)ops = &mrvl_mtr_ops;
1926
1927         return 0;
1928 }
1929
1930 /**
1931  * DPDK callback to get rte_tm callbacks.
1932  *
1933  * @param dev
1934  *   Pointer to the device structure.
1935  * @param ops
1936  *   Pointer to pass the tm ops.
1937  *
1938  * @return
1939  *   Always 0.
1940  */
1941 static int
1942 mrvl_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
1943 {
1944         *(const void **)ops = &mrvl_tm_ops;
1945
1946         return 0;
1947 }
1948
1949 static const struct eth_dev_ops mrvl_ops = {
1950         .dev_configure = mrvl_dev_configure,
1951         .dev_start = mrvl_dev_start,
1952         .dev_stop = mrvl_dev_stop,
1953         .dev_set_link_up = mrvl_dev_set_link_up,
1954         .dev_set_link_down = mrvl_dev_set_link_down,
1955         .dev_close = mrvl_dev_close,
1956         .link_update = mrvl_link_update,
1957         .promiscuous_enable = mrvl_promiscuous_enable,
1958         .allmulticast_enable = mrvl_allmulticast_enable,
1959         .promiscuous_disable = mrvl_promiscuous_disable,
1960         .allmulticast_disable = mrvl_allmulticast_disable,
1961         .mac_addr_remove = mrvl_mac_addr_remove,
1962         .mac_addr_add = mrvl_mac_addr_add,
1963         .mac_addr_set = mrvl_mac_addr_set,
1964         .mtu_set = mrvl_mtu_set,
1965         .stats_get = mrvl_stats_get,
1966         .stats_reset = mrvl_stats_reset,
1967         .xstats_get = mrvl_xstats_get,
1968         .xstats_reset = mrvl_xstats_reset,
1969         .xstats_get_names = mrvl_xstats_get_names,
1970         .dev_infos_get = mrvl_dev_infos_get,
1971         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
1972         .rxq_info_get = mrvl_rxq_info_get,
1973         .txq_info_get = mrvl_txq_info_get,
1974         .vlan_filter_set = mrvl_vlan_filter_set,
1975         .tx_queue_start = mrvl_tx_queue_start,
1976         .tx_queue_stop = mrvl_tx_queue_stop,
1977         .rx_queue_setup = mrvl_rx_queue_setup,
1978         .rx_queue_release = mrvl_rx_queue_release,
1979         .tx_queue_setup = mrvl_tx_queue_setup,
1980         .tx_queue_release = mrvl_tx_queue_release,
1981         .flow_ctrl_get = mrvl_flow_ctrl_get,
1982         .flow_ctrl_set = mrvl_flow_ctrl_set,
1983         .rss_hash_update = mrvl_rss_hash_update,
1984         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1985         .filter_ctrl = mrvl_eth_filter_ctrl,
1986         .mtr_ops_get = mrvl_mtr_ops_get,
1987         .tm_ops_get = mrvl_tm_ops_get,
1988 };
1989
1990 /**
1991  * Return packet type information and l3/l4 offsets.
1992  *
1993  * @param desc
1994  *   Pointer to the received packet descriptor.
1995  * @param l3_offset
1996  *   l3 packet offset.
1997  * @param l4_offset
1998  *   l4 packet offset.
1999  *
2000  * @return
2001  *   Packet type information.
2002  */
2003 static inline uint64_t
2004 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
2005                                     uint8_t *l3_offset, uint8_t *l4_offset)
2006 {
2007         enum pp2_inq_l3_type l3_type;
2008         enum pp2_inq_l4_type l4_type;
2009         enum pp2_inq_vlan_tag vlan_tag;
2010         uint64_t packet_type;
2011
2012         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
2013         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
2014         pp2_ppio_inq_desc_get_vlan_tag(desc, &vlan_tag);
2015
2016         packet_type = RTE_PTYPE_L2_ETHER;
2017
2018         switch (vlan_tag) {
2019         case PP2_INQ_VLAN_TAG_SINGLE:
2020                 packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
2021                 break;
2022         case PP2_INQ_VLAN_TAG_DOUBLE:
2023         case PP2_INQ_VLAN_TAG_TRIPLE:
2024                 packet_type |= RTE_PTYPE_L2_ETHER_QINQ;
2025                 break;
2026         default:
2027                 break;
2028         }
2029
2030         switch (l3_type) {
2031         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
2032                 packet_type |= RTE_PTYPE_L3_IPV4;
2033                 break;
2034         case PP2_INQ_L3_TYPE_IPV4_OK:
2035                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
2036                 break;
2037         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
2038                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
2039                 break;
2040         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
2041                 packet_type |= RTE_PTYPE_L3_IPV6;
2042                 break;
2043         case PP2_INQ_L3_TYPE_IPV6_EXT:
2044                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
2045                 break;
2046         case PP2_INQ_L3_TYPE_ARP:
2047                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
2048                 /*
2049                  * In case of ARP l4_offset is set to wrong value.
2050                  * Set it to proper one so that later on mbuf->l3_len can be
2051                  * calculated subtracting l4_offset and l3_offset.
2052                  */
2053                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
2054                 break;
2055         default:
2056                 MRVL_LOG(DEBUG, "Failed to recognise l3 packet type");
2057                 break;
2058         }
2059
2060         switch (l4_type) {
2061         case PP2_INQ_L4_TYPE_TCP:
2062                 packet_type |= RTE_PTYPE_L4_TCP;
2063                 break;
2064         case PP2_INQ_L4_TYPE_UDP:
2065                 packet_type |= RTE_PTYPE_L4_UDP;
2066                 break;
2067         default:
2068                 MRVL_LOG(DEBUG, "Failed to recognise l4 packet type");
2069                 break;
2070         }
2071
2072         return packet_type;
2073 }
2074
2075 /**
2076  * Get offload information from the received packet descriptor.
2077  *
2078  * @param desc
2079  *   Pointer to the received packet descriptor.
2080  *
2081  * @return
2082  *   Mbuf offload flags.
2083  */
2084 static inline uint64_t
2085 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
2086 {
2087         uint64_t flags;
2088         enum pp2_inq_desc_status status;
2089
2090         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
2091         if (unlikely(status != PP2_DESC_ERR_OK))
2092                 flags = PKT_RX_IP_CKSUM_BAD;
2093         else
2094                 flags = PKT_RX_IP_CKSUM_GOOD;
2095
2096         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
2097         if (unlikely(status != PP2_DESC_ERR_OK))
2098                 flags |= PKT_RX_L4_CKSUM_BAD;
2099         else
2100                 flags |= PKT_RX_L4_CKSUM_GOOD;
2101
2102         return flags;
2103 }
2104
2105 /**
2106  * DPDK callback for receive.
2107  *
2108  * @param rxq
2109  *   Generic pointer to the receive queue.
2110  * @param rx_pkts
2111  *   Array to store received packets.
2112  * @param nb_pkts
2113  *   Maximum number of packets in array.
2114  *
2115  * @return
2116  *   Number of packets successfully received.
2117  */
2118 static uint16_t
2119 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2120 {
2121         struct mrvl_rxq *q = rxq;
2122         struct pp2_ppio_desc descs[nb_pkts];
2123         struct pp2_bpool *bpool;
2124         int i, ret, rx_done = 0;
2125         int num;
2126         struct pp2_hif *hif;
2127         unsigned int core_id = rte_lcore_id();
2128
2129         hif = mrvl_get_hif(q->priv, core_id);
2130
2131         if (unlikely(!q->priv->ppio || !hif))
2132                 return 0;
2133
2134         bpool = q->priv->bpool;
2135
2136         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
2137                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
2138         if (unlikely(ret < 0)) {
2139                 MRVL_LOG(ERR, "Failed to receive packets");
2140                 return 0;
2141         }
2142         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
2143
2144         for (i = 0; i < nb_pkts; i++) {
2145                 struct rte_mbuf *mbuf;
2146                 uint8_t l3_offset, l4_offset;
2147                 enum pp2_inq_desc_status status;
2148                 uint64_t addr;
2149
2150                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2151                         struct pp2_ppio_desc *pref_desc;
2152                         u64 pref_addr;
2153
2154                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
2155                         pref_addr = cookie_addr_high |
2156                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
2157                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
2158                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
2159                 }
2160
2161                 addr = cookie_addr_high |
2162                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
2163                 mbuf = (struct rte_mbuf *)addr;
2164                 rte_pktmbuf_reset(mbuf);
2165
2166                 /* drop packet in case of mac, overrun or resource error */
2167                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
2168                 if (unlikely(status != PP2_DESC_ERR_OK)) {
2169                         struct pp2_buff_inf binf = {
2170                                 .addr = rte_mbuf_data_iova_default(mbuf),
2171                                 .cookie = (uint64_t)mbuf,
2172                         };
2173
2174                         pp2_bpool_put_buff(hif, bpool, &binf);
2175                         mrvl_port_bpool_size
2176                                 [bpool->pp2_id][bpool->id][core_id]++;
2177                         q->drop_mac++;
2178                         continue;
2179                 }
2180
2181                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
2182                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
2183                 mbuf->data_len = mbuf->pkt_len;
2184                 mbuf->port = q->port_id;
2185                 mbuf->packet_type =
2186                         mrvl_desc_to_packet_type_and_offset(&descs[i],
2187                                                             &l3_offset,
2188                                                             &l4_offset);
2189                 mbuf->l2_len = l3_offset;
2190                 mbuf->l3_len = l4_offset - l3_offset;
2191
2192                 if (likely(q->cksum_enabled))
2193                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
2194
2195                 rx_pkts[rx_done++] = mbuf;
2196                 q->bytes_recv += mbuf->pkt_len;
2197         }
2198
2199         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
2200                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
2201
2202                 if (unlikely(num <= q->priv->bpool_min_size ||
2203                              (!rx_done && num < q->priv->bpool_init_size))) {
2204                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
2205                         if (ret)
2206                                 MRVL_LOG(ERR, "Failed to fill bpool");
2207                 } else if (unlikely(num > q->priv->bpool_max_size)) {
2208                         int i;
2209                         int pkt_to_remove = num - q->priv->bpool_init_size;
2210                         struct rte_mbuf *mbuf;
2211                         struct pp2_buff_inf buff;
2212
2213                         MRVL_LOG(DEBUG,
2214                                 "port-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)",
2215                                 bpool->pp2_id, q->priv->ppio->port_id,
2216                                 bpool->id, pkt_to_remove, num,
2217                                 q->priv->bpool_init_size);
2218
2219                         for (i = 0; i < pkt_to_remove; i++) {
2220                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
2221                                 if (ret)
2222                                         break;
2223                                 mbuf = (struct rte_mbuf *)
2224                                         (cookie_addr_high | buff.cookie);
2225                                 rte_pktmbuf_free(mbuf);
2226                         }
2227                         mrvl_port_bpool_size
2228                                 [bpool->pp2_id][bpool->id][core_id] -= i;
2229                 }
2230                 rte_spinlock_unlock(&q->priv->lock);
2231         }
2232
2233         return rx_done;
2234 }
2235
2236 /**
2237  * Prepare offload information.
2238  *
2239  * @param ol_flags
2240  *   Offload flags.
2241  * @param packet_type
2242  *   Packet type bitfield.
2243  * @param l3_type
2244  *   Pointer to the pp2_ouq_l3_type structure.
2245  * @param l4_type
2246  *   Pointer to the pp2_outq_l4_type structure.
2247  * @param gen_l3_cksum
2248  *   Will be set to 1 in case l3 checksum is computed.
2249  * @param l4_cksum
2250  *   Will be set to 1 in case l4 checksum is computed.
2251  *
2252  * @return
2253  *   0 on success, negative error value otherwise.
2254  */
2255 static inline int
2256 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
2257                         enum pp2_outq_l3_type *l3_type,
2258                         enum pp2_outq_l4_type *l4_type,
2259                         int *gen_l3_cksum,
2260                         int *gen_l4_cksum)
2261 {
2262         /*
2263          * Based on ol_flags prepare information
2264          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
2265          * for offloading.
2266          */
2267         if (ol_flags & PKT_TX_IPV4) {
2268                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
2269                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
2270         } else if (ol_flags & PKT_TX_IPV6) {
2271                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
2272                 /* no checksum for ipv6 header */
2273                 *gen_l3_cksum = 0;
2274         } else {
2275                 /* if something different then stop processing */
2276                 return -1;
2277         }
2278
2279         ol_flags &= PKT_TX_L4_MASK;
2280         if ((packet_type & RTE_PTYPE_L4_TCP) &&
2281             ol_flags == PKT_TX_TCP_CKSUM) {
2282                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
2283                 *gen_l4_cksum = 1;
2284         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
2285                    ol_flags == PKT_TX_UDP_CKSUM) {
2286                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
2287                 *gen_l4_cksum = 1;
2288         } else {
2289                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
2290                 /* no checksum for other type */
2291                 *gen_l4_cksum = 0;
2292         }
2293
2294         return 0;
2295 }
2296
2297 /**
2298  * Release already sent buffers to bpool (buffer-pool).
2299  *
2300  * @param ppio
2301  *   Pointer to the port structure.
2302  * @param hif
2303  *   Pointer to the MUSDK hardware interface.
2304  * @param sq
2305  *   Pointer to the shadow queue.
2306  * @param qid
2307  *   Queue id number.
2308  * @param force
2309  *   Force releasing packets.
2310  */
2311 static inline void
2312 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
2313                        unsigned int core_id, struct mrvl_shadow_txq *sq,
2314                        int qid, int force)
2315 {
2316         struct buff_release_entry *entry;
2317         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
2318         int i;
2319
2320         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
2321
2322         sq->num_to_release += nb_done;
2323
2324         if (likely(!force &&
2325                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
2326                 return;
2327
2328         nb_done = sq->num_to_release;
2329         sq->num_to_release = 0;
2330
2331         for (i = 0; i < nb_done; i++) {
2332                 entry = &sq->ent[sq->tail + num];
2333                 if (unlikely(!entry->buff.addr)) {
2334                         MRVL_LOG(ERR,
2335                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!",
2336                                 sq->tail, (u64)entry->buff.cookie,
2337                                 (u64)entry->buff.addr);
2338                         skip_bufs = 1;
2339                         goto skip;
2340                 }
2341
2342                 if (unlikely(!entry->bpool)) {
2343                         struct rte_mbuf *mbuf;
2344
2345                         mbuf = (struct rte_mbuf *)
2346                                (cookie_addr_high | entry->buff.cookie);
2347                         rte_pktmbuf_free(mbuf);
2348                         skip_bufs = 1;
2349                         goto skip;
2350                 }
2351
2352                 mrvl_port_bpool_size
2353                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
2354                 num++;
2355                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
2356                         goto skip;
2357                 continue;
2358 skip:
2359                 if (likely(num))
2360                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2361                 num += skip_bufs;
2362                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2363                 sq->size -= num;
2364                 num = 0;
2365                 skip_bufs = 0;
2366         }
2367
2368         if (likely(num)) {
2369                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2370                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2371                 sq->size -= num;
2372         }
2373 }
2374
2375 /**
2376  * DPDK callback for transmit.
2377  *
2378  * @param txq
2379  *   Generic pointer transmit queue.
2380  * @param tx_pkts
2381  *   Packets to transmit.
2382  * @param nb_pkts
2383  *   Number of packets in array.
2384  *
2385  * @return
2386  *   Number of packets successfully transmitted.
2387  */
2388 static uint16_t
2389 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2390 {
2391         struct mrvl_txq *q = txq;
2392         struct mrvl_shadow_txq *sq;
2393         struct pp2_hif *hif;
2394         struct pp2_ppio_desc descs[nb_pkts];
2395         unsigned int core_id = rte_lcore_id();
2396         int i, ret, bytes_sent = 0;
2397         uint16_t num, sq_free_size;
2398         uint64_t addr;
2399
2400         hif = mrvl_get_hif(q->priv, core_id);
2401         sq = &q->shadow_txqs[core_id];
2402
2403         if (unlikely(!q->priv->ppio || !hif))
2404                 return 0;
2405
2406         if (sq->size)
2407                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2408                                        sq, q->queue_id, 0);
2409
2410         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2411         if (unlikely(nb_pkts > sq_free_size)) {
2412                 MRVL_LOG(DEBUG,
2413                         "No room in shadow queue for %d packets! %d packets will be sent.",
2414                         nb_pkts, sq_free_size);
2415                 nb_pkts = sq_free_size;
2416         }
2417
2418         for (i = 0; i < nb_pkts; i++) {
2419                 struct rte_mbuf *mbuf = tx_pkts[i];
2420                 int gen_l3_cksum, gen_l4_cksum;
2421                 enum pp2_outq_l3_type l3_type;
2422                 enum pp2_outq_l4_type l4_type;
2423
2424                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2425                         struct rte_mbuf *pref_pkt_hdr;
2426
2427                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2428                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2429                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2430                 }
2431
2432                 sq->ent[sq->head].buff.cookie = (uint64_t)mbuf;
2433                 sq->ent[sq->head].buff.addr =
2434                         rte_mbuf_data_iova_default(mbuf);
2435                 sq->ent[sq->head].bpool =
2436                         (unlikely(mbuf->port >= RTE_MAX_ETHPORTS ||
2437                          mbuf->refcnt > 1)) ? NULL :
2438                          mrvl_port_to_bpool_lookup[mbuf->port];
2439                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
2440                 sq->size++;
2441
2442                 pp2_ppio_outq_desc_reset(&descs[i]);
2443                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
2444                                                  rte_pktmbuf_iova(mbuf));
2445                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
2446                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
2447                                                rte_pktmbuf_pkt_len(mbuf));
2448
2449                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2450                 /*
2451                  * in case unsupported ol_flags were passed
2452                  * do not update descriptor offload information
2453                  */
2454                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
2455                                               &l3_type, &l4_type, &gen_l3_cksum,
2456                                               &gen_l4_cksum);
2457                 if (unlikely(ret))
2458                         continue;
2459
2460                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2461                                                   mbuf->l2_len,
2462                                                   mbuf->l2_len + mbuf->l3_len,
2463                                                   gen_l3_cksum, gen_l4_cksum);
2464         }
2465
2466         num = nb_pkts;
2467         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2468         /* number of packets that were not sent */
2469         if (unlikely(num > nb_pkts)) {
2470                 for (i = nb_pkts; i < num; i++) {
2471                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2472                                 MRVL_PP2_TX_SHADOWQ_MASK;
2473                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
2474                         bytes_sent -=
2475                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2476                 }
2477                 sq->size -= num - nb_pkts;
2478         }
2479
2480         q->bytes_sent += bytes_sent;
2481
2482         return nb_pkts;
2483 }
2484
2485 /**
2486  * Initialize packet processor.
2487  *
2488  * @return
2489  *   0 on success, negative error value otherwise.
2490  */
2491 static int
2492 mrvl_init_pp2(void)
2493 {
2494         struct pp2_init_params init_params;
2495
2496         memset(&init_params, 0, sizeof(init_params));
2497         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
2498         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
2499         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
2500
2501         return pp2_init(&init_params);
2502 }
2503
2504 /**
2505  * Deinitialize packet processor.
2506  *
2507  * @return
2508  *   0 on success, negative error value otherwise.
2509  */
2510 static void
2511 mrvl_deinit_pp2(void)
2512 {
2513         pp2_deinit();
2514 }
2515
2516 /**
2517  * Create private device structure.
2518  *
2519  * @param dev_name
2520  *   Pointer to the port name passed in the initialization parameters.
2521  *
2522  * @return
2523  *   Pointer to the newly allocated private device structure.
2524  */
2525 static struct mrvl_priv *
2526 mrvl_priv_create(const char *dev_name)
2527 {
2528         struct pp2_bpool_params bpool_params;
2529         char match[MRVL_MATCH_LEN];
2530         struct mrvl_priv *priv;
2531         int ret, bpool_bit;
2532
2533         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
2534         if (!priv)
2535                 return NULL;
2536
2537         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
2538                                        &priv->pp_id, &priv->ppio_id);
2539         if (ret)
2540                 goto out_free_priv;
2541
2542         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
2543                                      PP2_BPOOL_NUM_POOLS);
2544         if (bpool_bit < 0)
2545                 goto out_free_priv;
2546         priv->bpool_bit = bpool_bit;
2547
2548         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
2549                  priv->bpool_bit);
2550         memset(&bpool_params, 0, sizeof(bpool_params));
2551         bpool_params.match = match;
2552         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
2553         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
2554         if (ret)
2555                 goto out_clear_bpool_bit;
2556
2557         priv->ppio_params.type = PP2_PPIO_T_NIC;
2558         rte_spinlock_init(&priv->lock);
2559
2560         return priv;
2561 out_clear_bpool_bit:
2562         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2563 out_free_priv:
2564         rte_free(priv);
2565         return NULL;
2566 }
2567
2568 /**
2569  * Create device representing Ethernet port.
2570  *
2571  * @param name
2572  *   Pointer to the port's name.
2573  *
2574  * @return
2575  *   0 on success, negative error value otherwise.
2576  */
2577 static int
2578 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
2579 {
2580         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
2581         struct rte_eth_dev *eth_dev;
2582         struct mrvl_priv *priv;
2583         struct ifreq req;
2584
2585         eth_dev = rte_eth_dev_allocate(name);
2586         if (!eth_dev)
2587                 return -ENOMEM;
2588
2589         priv = mrvl_priv_create(name);
2590         if (!priv) {
2591                 ret = -ENOMEM;
2592                 goto out_free_dev;
2593         }
2594
2595         eth_dev->data->mac_addrs =
2596                 rte_zmalloc("mac_addrs",
2597                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2598         if (!eth_dev->data->mac_addrs) {
2599                 MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
2600                 ret = -ENOMEM;
2601                 goto out_free_priv;
2602         }
2603
2604         memset(&req, 0, sizeof(req));
2605         strcpy(req.ifr_name, name);
2606         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2607         if (ret)
2608                 goto out_free_mac;
2609
2610         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2611                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2612
2613         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2614         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
2615         eth_dev->data->kdrv = RTE_KDRV_NONE;
2616         eth_dev->data->dev_private = priv;
2617         eth_dev->device = &vdev->device;
2618         eth_dev->dev_ops = &mrvl_ops;
2619
2620         rte_eth_dev_probing_finish(eth_dev);
2621         return 0;
2622 out_free_mac:
2623         rte_free(eth_dev->data->mac_addrs);
2624 out_free_dev:
2625         rte_eth_dev_release_port(eth_dev);
2626 out_free_priv:
2627         rte_free(priv);
2628
2629         return ret;
2630 }
2631
2632 /**
2633  * Cleanup previously created device representing Ethernet port.
2634  *
2635  * @param name
2636  *   Pointer to the port name.
2637  */
2638 static void
2639 mrvl_eth_dev_destroy(const char *name)
2640 {
2641         struct rte_eth_dev *eth_dev;
2642         struct mrvl_priv *priv;
2643
2644         eth_dev = rte_eth_dev_allocated(name);
2645         if (!eth_dev)
2646                 return;
2647
2648         priv = eth_dev->data->dev_private;
2649         pp2_bpool_deinit(priv->bpool);
2650         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2651         rte_free(priv);
2652         rte_free(eth_dev->data->mac_addrs);
2653         rte_eth_dev_release_port(eth_dev);
2654 }
2655
2656 /**
2657  * Callback used by rte_kvargs_process() during argument parsing.
2658  *
2659  * @param key
2660  *   Pointer to the parsed key (unused).
2661  * @param value
2662  *   Pointer to the parsed value.
2663  * @param extra_args
2664  *   Pointer to the extra arguments which contains address of the
2665  *   table of pointers to parsed interface names.
2666  *
2667  * @return
2668  *   Always 0.
2669  */
2670 static int
2671 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2672                  void *extra_args)
2673 {
2674         struct mrvl_ifnames *ifnames = extra_args;
2675
2676         ifnames->names[ifnames->idx++] = value;
2677
2678         return 0;
2679 }
2680
2681 /**
2682  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2683  */
2684 static void
2685 mrvl_deinit_hifs(void)
2686 {
2687         int i;
2688
2689         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
2690                 if (hifs[i])
2691                         pp2_hif_deinit(hifs[i]);
2692         }
2693         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
2694         memset(hifs, 0, sizeof(hifs));
2695 }
2696
2697 /**
2698  * DPDK callback to register the virtual device.
2699  *
2700  * @param vdev
2701  *   Pointer to the virtual device.
2702  *
2703  * @return
2704  *   0 on success, negative error value otherwise.
2705  */
2706 static int
2707 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2708 {
2709         struct rte_kvargs *kvlist;
2710         struct mrvl_ifnames ifnames;
2711         int ret = -EINVAL;
2712         uint32_t i, ifnum, cfgnum;
2713         const char *params;
2714
2715         params = rte_vdev_device_args(vdev);
2716         if (!params)
2717                 return -EINVAL;
2718
2719         kvlist = rte_kvargs_parse(params, valid_args);
2720         if (!kvlist)
2721                 return -EINVAL;
2722
2723         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2724         if (ifnum > RTE_DIM(ifnames.names))
2725                 goto out_free_kvlist;
2726
2727         ifnames.idx = 0;
2728         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2729                            mrvl_get_ifnames, &ifnames);
2730
2731
2732         /*
2733          * The below system initialization should be done only once,
2734          * on the first provided configuration file
2735          */
2736         if (!mrvl_qos_cfg) {
2737                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2738                 MRVL_LOG(INFO, "Parsing config file!");
2739                 if (cfgnum > 1) {
2740                         MRVL_LOG(ERR, "Cannot handle more than one config file!");
2741                         goto out_free_kvlist;
2742                 } else if (cfgnum == 1) {
2743                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2744                                            mrvl_get_qoscfg, &mrvl_qos_cfg);
2745                 }
2746         }
2747
2748         if (mrvl_dev_num)
2749                 goto init_devices;
2750
2751         MRVL_LOG(INFO, "Perform MUSDK initializations");
2752
2753         ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist);
2754         if (ret)
2755                 goto out_free_kvlist;
2756
2757         ret = mrvl_init_pp2();
2758         if (ret) {
2759                 MRVL_LOG(ERR, "Failed to init PP!");
2760                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2761                 goto out_free_kvlist;
2762         }
2763
2764         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2765         memset(mrvl_port_to_bpool_lookup, 0, sizeof(mrvl_port_to_bpool_lookup));
2766
2767         mrvl_lcore_first = RTE_MAX_LCORE;
2768         mrvl_lcore_last = 0;
2769
2770 init_devices:
2771         for (i = 0; i < ifnum; i++) {
2772                 MRVL_LOG(INFO, "Creating %s", ifnames.names[i]);
2773                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
2774                 if (ret)
2775                         goto out_cleanup;
2776         }
2777         mrvl_dev_num += ifnum;
2778
2779         rte_kvargs_free(kvlist);
2780
2781         return 0;
2782 out_cleanup:
2783         for (; i > 0; i--)
2784                 mrvl_eth_dev_destroy(ifnames.names[i]);
2785
2786         if (mrvl_dev_num == 0) {
2787                 mrvl_deinit_pp2();
2788                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2789         }
2790 out_free_kvlist:
2791         rte_kvargs_free(kvlist);
2792
2793         return ret;
2794 }
2795
2796 /**
2797  * DPDK callback to remove virtual device.
2798  *
2799  * @param vdev
2800  *   Pointer to the removed virtual device.
2801  *
2802  * @return
2803  *   0 on success, negative error value otherwise.
2804  */
2805 static int
2806 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2807 {
2808         int i;
2809         const char *name;
2810
2811         name = rte_vdev_device_name(vdev);
2812         if (!name)
2813                 return -EINVAL;
2814
2815         MRVL_LOG(INFO, "Removing %s", name);
2816
2817         RTE_ETH_FOREACH_DEV(i) { /* FIXME: removing all devices! */
2818                 char ifname[RTE_ETH_NAME_MAX_LEN];
2819
2820                 rte_eth_dev_get_name_by_port(i, ifname);
2821                 mrvl_eth_dev_destroy(ifname);
2822                 mrvl_dev_num--;
2823         }
2824
2825         if (mrvl_dev_num == 0) {
2826                 MRVL_LOG(INFO, "Perform MUSDK deinit");
2827                 mrvl_deinit_hifs();
2828                 mrvl_deinit_pp2();
2829                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2830         }
2831
2832         return 0;
2833 }
2834
2835 static struct rte_vdev_driver pmd_mrvl_drv = {
2836         .probe = rte_pmd_mrvl_probe,
2837         .remove = rte_pmd_mrvl_remove,
2838 };
2839
2840 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv);
2841 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2);
2842
2843 RTE_INIT(mrvl_init_log)
2844 {
2845         mrvl_logtype = rte_log_register("pmd.net.mvpp2");
2846         if (mrvl_logtype >= 0)
2847                 rte_log_set_level(mrvl_logtype, RTE_LOG_NOTICE);
2848 }