ethdev: make stats and xstats reset callbacks return int
[dpdk.git] / drivers / net / mvpp2 / mrvl_ethdev.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017 Marvell International Ltd.
3  * Copyright(c) 2017 Semihalf.
4  * All rights reserved.
5  */
6
7 #include <rte_string_fns.h>
8 #include <rte_ethdev_driver.h>
9 #include <rte_kvargs.h>
10 #include <rte_log.h>
11 #include <rte_malloc.h>
12 #include <rte_bus_vdev.h>
13
14 #include <fcntl.h>
15 #include <linux/ethtool.h>
16 #include <linux/sockios.h>
17 #include <net/if.h>
18 #include <net/if_arp.h>
19 #include <sys/ioctl.h>
20 #include <sys/socket.h>
21 #include <sys/stat.h>
22 #include <sys/types.h>
23
24 #include <rte_mvep_common.h>
25 #include "mrvl_ethdev.h"
26 #include "mrvl_qos.h"
27 #include "mrvl_flow.h"
28 #include "mrvl_mtr.h"
29 #include "mrvl_tm.h"
30
31 /* bitmask with reserved hifs */
32 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
33 /* bitmask with reserved bpools */
34 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
35 /* bitmask with reserved kernel RSS tables */
36 #define MRVL_MUSDK_RSS_RESERVED 0x01
37 /* maximum number of available hifs */
38 #define MRVL_MUSDK_HIFS_MAX 9
39
40 /* prefetch shift */
41 #define MRVL_MUSDK_PREFETCH_SHIFT 2
42
43 /* TCAM has 25 entries reserved for uc/mc filter entries */
44 #define MRVL_MAC_ADDRS_MAX 25
45 #define MRVL_MATCH_LEN 16
46 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
47 /* Maximum allowable packet size */
48 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
49
50 #define MRVL_IFACE_NAME_ARG "iface"
51 #define MRVL_CFG_ARG "cfg"
52
53 #define MRVL_BURST_SIZE 64
54
55 #define MRVL_ARP_LENGTH 28
56
57 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
58 #define MRVL_COOKIE_HIGH_ADDR_MASK 0xffffff0000000000
59
60 /** Port Rx offload capabilities */
61 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
62                           DEV_RX_OFFLOAD_JUMBO_FRAME | \
63                           DEV_RX_OFFLOAD_CHECKSUM)
64
65 /** Port Tx offloads capabilities */
66 #define MRVL_TX_OFFLOADS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
67                           DEV_TX_OFFLOAD_UDP_CKSUM | \
68                           DEV_TX_OFFLOAD_TCP_CKSUM | \
69                           DEV_TX_OFFLOAD_MULTI_SEGS)
70
71 static const char * const valid_args[] = {
72         MRVL_IFACE_NAME_ARG,
73         MRVL_CFG_ARG,
74         NULL
75 };
76
77 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
78 static struct pp2_hif *hifs[RTE_MAX_LCORE];
79 static int used_bpools[PP2_NUM_PKT_PROC] = {
80         [0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED
81 };
82
83 static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
84 static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
85 static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
86
87 int mrvl_logtype;
88
89 struct mrvl_ifnames {
90         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
91         int idx;
92 };
93
94 /*
95  * To use buffer harvesting based on loopback port shadow queue structure
96  * was introduced for buffers information bookkeeping.
97  *
98  * Before sending the packet, related buffer information (pp2_buff_inf) is
99  * stored in shadow queue. After packet is transmitted no longer used
100  * packet buffer is released back to it's original hardware pool,
101  * on condition it originated from interface.
102  * In case it  was generated by application itself i.e: mbuf->port field is
103  * 0xff then its released to software mempool.
104  */
105 struct mrvl_shadow_txq {
106         int head;           /* write index - used when sending buffers */
107         int tail;           /* read index - used when releasing buffers */
108         u16 size;           /* queue occupied size */
109         u16 num_to_release; /* number of descriptors sent, that can be
110                              * released
111                              */
112         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
113 };
114
115 struct mrvl_rxq {
116         struct mrvl_priv *priv;
117         struct rte_mempool *mp;
118         int queue_id;
119         int port_id;
120         int cksum_enabled;
121         uint64_t bytes_recv;
122         uint64_t drop_mac;
123 };
124
125 struct mrvl_txq {
126         struct mrvl_priv *priv;
127         int queue_id;
128         int port_id;
129         uint64_t bytes_sent;
130         struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE];
131         int tx_deferred_start;
132 };
133
134 static int mrvl_lcore_first;
135 static int mrvl_lcore_last;
136 static int mrvl_dev_num;
137
138 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
139 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
140                         struct pp2_hif *hif, unsigned int core_id,
141                         struct mrvl_shadow_txq *sq, int qid, int force);
142
143 static uint16_t mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
144                                   uint16_t nb_pkts);
145 static uint16_t mrvl_tx_sg_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
146                                      uint16_t nb_pkts);
147 static int rte_pmd_mrvl_remove(struct rte_vdev_device *vdev);
148 static void mrvl_deinit_pp2(void);
149 static void mrvl_deinit_hifs(void);
150
151
152 #define MRVL_XSTATS_TBL_ENTRY(name) { \
153         #name, offsetof(struct pp2_ppio_statistics, name),      \
154         sizeof(((struct pp2_ppio_statistics *)0)->name)         \
155 }
156
157 /* Table with xstats data */
158 static struct {
159         const char *name;
160         unsigned int offset;
161         unsigned int size;
162 } mrvl_xstats_tbl[] = {
163         MRVL_XSTATS_TBL_ENTRY(rx_bytes),
164         MRVL_XSTATS_TBL_ENTRY(rx_packets),
165         MRVL_XSTATS_TBL_ENTRY(rx_unicast_packets),
166         MRVL_XSTATS_TBL_ENTRY(rx_errors),
167         MRVL_XSTATS_TBL_ENTRY(rx_fullq_dropped),
168         MRVL_XSTATS_TBL_ENTRY(rx_bm_dropped),
169         MRVL_XSTATS_TBL_ENTRY(rx_early_dropped),
170         MRVL_XSTATS_TBL_ENTRY(rx_fifo_dropped),
171         MRVL_XSTATS_TBL_ENTRY(rx_cls_dropped),
172         MRVL_XSTATS_TBL_ENTRY(tx_bytes),
173         MRVL_XSTATS_TBL_ENTRY(tx_packets),
174         MRVL_XSTATS_TBL_ENTRY(tx_unicast_packets),
175         MRVL_XSTATS_TBL_ENTRY(tx_errors)
176 };
177
178 static inline void
179 mrvl_fill_shadowq(struct mrvl_shadow_txq *sq, struct rte_mbuf *buf)
180 {
181         sq->ent[sq->head].buff.cookie = (uint64_t)buf;
182         sq->ent[sq->head].buff.addr = buf ?
183                 rte_mbuf_data_iova_default(buf) : 0;
184
185         sq->ent[sq->head].bpool =
186                 (unlikely(!buf || buf->port >= RTE_MAX_ETHPORTS ||
187                  buf->refcnt > 1)) ? NULL :
188                  mrvl_port_to_bpool_lookup[buf->port];
189
190         sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
191         sq->size++;
192 }
193
194 static inline void
195 mrvl_fill_desc(struct pp2_ppio_desc *desc, struct rte_mbuf *buf)
196 {
197         pp2_ppio_outq_desc_reset(desc);
198         pp2_ppio_outq_desc_set_phys_addr(desc, rte_pktmbuf_iova(buf));
199         pp2_ppio_outq_desc_set_pkt_offset(desc, 0);
200         pp2_ppio_outq_desc_set_pkt_len(desc, rte_pktmbuf_data_len(buf));
201 }
202
203 static inline int
204 mrvl_get_bpool_size(int pp2_id, int pool_id)
205 {
206         int i;
207         int size = 0;
208
209         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
210                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
211
212         return size;
213 }
214
215 static inline int
216 mrvl_reserve_bit(int *bitmap, int max)
217 {
218         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
219
220         if (n >= max)
221                 return -1;
222
223         *bitmap |= 1 << n;
224
225         return n;
226 }
227
228 static int
229 mrvl_init_hif(int core_id)
230 {
231         struct pp2_hif_params params;
232         char match[MRVL_MATCH_LEN];
233         int ret;
234
235         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
236         if (ret < 0) {
237                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
238                 return ret;
239         }
240
241         snprintf(match, sizeof(match), "hif-%d", ret);
242         memset(&params, 0, sizeof(params));
243         params.match = match;
244         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
245         ret = pp2_hif_init(&params, &hifs[core_id]);
246         if (ret) {
247                 MRVL_LOG(ERR, "Failed to initialize hif %d", core_id);
248                 return ret;
249         }
250
251         return 0;
252 }
253
254 static inline struct pp2_hif*
255 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
256 {
257         int ret;
258
259         if (likely(hifs[core_id] != NULL))
260                 return hifs[core_id];
261
262         rte_spinlock_lock(&priv->lock);
263
264         ret = mrvl_init_hif(core_id);
265         if (ret < 0) {
266                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
267                 goto out;
268         }
269
270         if (core_id < mrvl_lcore_first)
271                 mrvl_lcore_first = core_id;
272
273         if (core_id > mrvl_lcore_last)
274                 mrvl_lcore_last = core_id;
275 out:
276         rte_spinlock_unlock(&priv->lock);
277
278         return hifs[core_id];
279 }
280
281 /**
282  * Set tx burst function according to offload flag
283  *
284  * @param dev
285  *   Pointer to Ethernet device structure.
286  */
287 static void
288 mrvl_set_tx_function(struct rte_eth_dev *dev)
289 {
290         struct mrvl_priv *priv = dev->data->dev_private;
291
292         /* Use a simple Tx queue (no offloads, no multi segs) if possible */
293         if (priv->multiseg) {
294                 RTE_LOG(INFO, PMD, "Using multi-segment tx callback\n");
295                 dev->tx_pkt_burst = mrvl_tx_sg_pkt_burst;
296         } else {
297                 RTE_LOG(INFO, PMD, "Using single-segment tx callback\n");
298                 dev->tx_pkt_burst = mrvl_tx_pkt_burst;
299         }
300 }
301
302 /**
303  * Configure rss based on dpdk rss configuration.
304  *
305  * @param priv
306  *   Pointer to private structure.
307  * @param rss_conf
308  *   Pointer to RSS configuration.
309  *
310  * @return
311  *   0 on success, negative error value otherwise.
312  */
313 static int
314 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
315 {
316         if (rss_conf->rss_key)
317                 MRVL_LOG(WARNING, "Changing hash key is not supported");
318
319         if (rss_conf->rss_hf == 0) {
320                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
321         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
322                 priv->ppio_params.inqs_params.hash_type =
323                         PP2_PPIO_HASH_T_2_TUPLE;
324         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
325                 priv->ppio_params.inqs_params.hash_type =
326                         PP2_PPIO_HASH_T_5_TUPLE;
327                 priv->rss_hf_tcp = 1;
328         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
329                 priv->ppio_params.inqs_params.hash_type =
330                         PP2_PPIO_HASH_T_5_TUPLE;
331                 priv->rss_hf_tcp = 0;
332         } else {
333                 return -EINVAL;
334         }
335
336         return 0;
337 }
338
339 /**
340  * Ethernet device configuration.
341  *
342  * Prepare the driver for a given number of TX and RX queues and
343  * configure RSS.
344  *
345  * @param dev
346  *   Pointer to Ethernet device structure.
347  *
348  * @return
349  *   0 on success, negative error value otherwise.
350  */
351 static int
352 mrvl_dev_configure(struct rte_eth_dev *dev)
353 {
354         struct mrvl_priv *priv = dev->data->dev_private;
355         int ret;
356
357         if (priv->ppio) {
358                 MRVL_LOG(INFO, "Device reconfiguration is not supported");
359                 return -EINVAL;
360         }
361
362         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
363             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
364                 MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
365                         dev->data->dev_conf.rxmode.mq_mode);
366                 return -EINVAL;
367         }
368
369         if (dev->data->dev_conf.rxmode.split_hdr_size) {
370                 MRVL_LOG(INFO, "Split headers not supported");
371                 return -EINVAL;
372         }
373
374         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
375                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
376                                  MRVL_PP2_ETH_HDRS_LEN;
377
378         if (dev->data->dev_conf.txmode.offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
379                 priv->multiseg = 1;
380
381         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
382                                   dev->data->nb_rx_queues);
383         if (ret < 0)
384                 return ret;
385
386         ret = mrvl_configure_txqs(priv, dev->data->port_id,
387                                   dev->data->nb_tx_queues);
388         if (ret < 0)
389                 return ret;
390
391         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
392         priv->ppio_params.maintain_stats = 1;
393         priv->nb_rx_queues = dev->data->nb_rx_queues;
394
395         ret = mrvl_tm_init(dev);
396         if (ret < 0)
397                 return ret;
398
399         if (dev->data->nb_rx_queues == 1 &&
400             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
401                 MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
402                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
403
404                 return 0;
405         }
406
407         return mrvl_configure_rss(priv,
408                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
409 }
410
411 /**
412  * DPDK callback to change the MTU.
413  *
414  * Setting the MTU affects hardware MRU (packets larger than the MRU
415  * will be dropped).
416  *
417  * @param dev
418  *   Pointer to Ethernet device structure.
419  * @param mtu
420  *   New MTU.
421  *
422  * @return
423  *   0 on success, negative error value otherwise.
424  */
425 static int
426 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
427 {
428         struct mrvl_priv *priv = dev->data->dev_private;
429         uint16_t mru;
430         uint16_t mbuf_data_size = 0; /* SW buffer size */
431         int ret;
432
433         mru = MRVL_PP2_MTU_TO_MRU(mtu);
434         /*
435          * min_rx_buf_size is equal to mbuf data size
436          * if pmd didn't set it differently
437          */
438         mbuf_data_size = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
439         /* Prevent PMD from:
440          * - setting mru greater than the mbuf size resulting in
441          * hw and sw buffer size mismatch
442          * - setting mtu that requires the support of scattered packets
443          * when this feature has not been enabled/supported so far
444          * (TODO check scattered_rx flag here once scattered RX is supported).
445          */
446         if (mru + MRVL_PKT_OFFS > mbuf_data_size) {
447                 mru = mbuf_data_size - MRVL_PKT_OFFS;
448                 mtu = MRVL_PP2_MRU_TO_MTU(mru);
449                 MRVL_LOG(WARNING, "MTU too big, max MTU possible limitted "
450                         "by current mbuf size: %u. Set MTU to %u, MRU to %u",
451                         mbuf_data_size, mtu, mru);
452         }
453
454         if (mtu < RTE_ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX) {
455                 MRVL_LOG(ERR, "Invalid MTU [%u] or MRU [%u]", mtu, mru);
456                 return -EINVAL;
457         }
458
459         dev->data->mtu = mtu;
460         dev->data->dev_conf.rxmode.max_rx_pkt_len = mru - MV_MH_SIZE;
461
462         if (!priv->ppio)
463                 return 0;
464
465         ret = pp2_ppio_set_mru(priv->ppio, mru);
466         if (ret) {
467                 MRVL_LOG(ERR, "Failed to change MRU");
468                 return ret;
469         }
470
471         ret = pp2_ppio_set_mtu(priv->ppio, mtu);
472         if (ret) {
473                 MRVL_LOG(ERR, "Failed to change MTU");
474                 return ret;
475         }
476
477         return 0;
478 }
479
480 /**
481  * DPDK callback to bring the link up.
482  *
483  * @param dev
484  *   Pointer to Ethernet device structure.
485  *
486  * @return
487  *   0 on success, negative error value otherwise.
488  */
489 static int
490 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
491 {
492         struct mrvl_priv *priv = dev->data->dev_private;
493         int ret;
494
495         if (!priv->ppio)
496                 return -EPERM;
497
498         ret = pp2_ppio_enable(priv->ppio);
499         if (ret)
500                 return ret;
501
502         /*
503          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
504          * as pp2_ppio_enable() changes port->t_mode from default 0 to
505          * PP2_TRAFFIC_INGRESS_EGRESS.
506          *
507          * Set mtu to default DPDK value here.
508          */
509         ret = mrvl_mtu_set(dev, dev->data->mtu);
510         if (ret)
511                 pp2_ppio_disable(priv->ppio);
512
513         return ret;
514 }
515
516 /**
517  * DPDK callback to bring the link down.
518  *
519  * @param dev
520  *   Pointer to Ethernet device structure.
521  *
522  * @return
523  *   0 on success, negative error value otherwise.
524  */
525 static int
526 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
527 {
528         struct mrvl_priv *priv = dev->data->dev_private;
529
530         if (!priv->ppio)
531                 return -EPERM;
532
533         return pp2_ppio_disable(priv->ppio);
534 }
535
536 /**
537  * DPDK callback to start tx queue.
538  *
539  * @param dev
540  *   Pointer to Ethernet device structure.
541  * @param queue_id
542  *   Transmit queue index.
543  *
544  * @return
545  *   0 on success, negative error value otherwise.
546  */
547 static int
548 mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
549 {
550         struct mrvl_priv *priv = dev->data->dev_private;
551         int ret;
552
553         if (!priv)
554                 return -EPERM;
555
556         /* passing 1 enables given tx queue */
557         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1);
558         if (ret) {
559                 MRVL_LOG(ERR, "Failed to start txq %d", queue_id);
560                 return ret;
561         }
562
563         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
564
565         return 0;
566 }
567
568 /**
569  * DPDK callback to stop tx queue.
570  *
571  * @param dev
572  *   Pointer to Ethernet device structure.
573  * @param queue_id
574  *   Transmit queue index.
575  *
576  * @return
577  *   0 on success, negative error value otherwise.
578  */
579 static int
580 mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
581 {
582         struct mrvl_priv *priv = dev->data->dev_private;
583         int ret;
584
585         if (!priv->ppio)
586                 return -EPERM;
587
588         /* passing 0 disables given tx queue */
589         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0);
590         if (ret) {
591                 MRVL_LOG(ERR, "Failed to stop txq %d", queue_id);
592                 return ret;
593         }
594
595         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
596
597         return 0;
598 }
599
600 /**
601  * DPDK callback to start the device.
602  *
603  * @param dev
604  *   Pointer to Ethernet device structure.
605  *
606  * @return
607  *   0 on success, negative errno value on failure.
608  */
609 static int
610 mrvl_dev_start(struct rte_eth_dev *dev)
611 {
612         struct mrvl_priv *priv = dev->data->dev_private;
613         char match[MRVL_MATCH_LEN];
614         int ret = 0, i, def_init_size;
615
616         if (priv->ppio)
617                 return mrvl_dev_set_link_up(dev);
618
619         snprintf(match, sizeof(match), "ppio-%d:%d",
620                  priv->pp_id, priv->ppio_id);
621         priv->ppio_params.match = match;
622
623         /*
624          * Calculate the minimum bpool size for refill feature as follows:
625          * 2 default burst sizes multiply by number of rx queues.
626          * If the bpool size will be below this value, new buffers will
627          * be added to the pool.
628          */
629         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
630
631         /* In case initial bpool size configured in queues setup is
632          * smaller than minimum size add more buffers
633          */
634         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
635         if (priv->bpool_init_size < def_init_size) {
636                 int buffs_to_add = def_init_size - priv->bpool_init_size;
637
638                 priv->bpool_init_size += buffs_to_add;
639                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
640                 if (ret)
641                         MRVL_LOG(ERR, "Failed to add buffers to bpool");
642         }
643
644         /*
645          * Calculate the maximum bpool size for refill feature as follows:
646          * maximum number of descriptors in rx queue multiply by number
647          * of rx queues plus minimum bpool size.
648          * In case the bpool size will exceed this value, superfluous buffers
649          * will be removed
650          */
651         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
652                                 priv->bpool_min_size;
653
654         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
655         if (ret) {
656                 MRVL_LOG(ERR, "Failed to init ppio");
657                 return ret;
658         }
659
660         /*
661          * In case there are some some stale uc/mc mac addresses flush them
662          * here. It cannot be done during mrvl_dev_close() as port information
663          * is already gone at that point (due to pp2_ppio_deinit() in
664          * mrvl_dev_stop()).
665          */
666         if (!priv->uc_mc_flushed) {
667                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
668                 if (ret) {
669                         MRVL_LOG(ERR,
670                                 "Failed to flush uc/mc filter list");
671                         goto out;
672                 }
673                 priv->uc_mc_flushed = 1;
674         }
675
676         if (!priv->vlan_flushed) {
677                 ret = pp2_ppio_flush_vlan(priv->ppio);
678                 if (ret) {
679                         MRVL_LOG(ERR, "Failed to flush vlan list");
680                         /*
681                          * TODO
682                          * once pp2_ppio_flush_vlan() is supported jump to out
683                          * goto out;
684                          */
685                 }
686                 priv->vlan_flushed = 1;
687         }
688         ret = mrvl_mtu_set(dev, dev->data->mtu);
689         if (ret)
690                 MRVL_LOG(ERR, "Failed to set MTU to %d", dev->data->mtu);
691
692         /* For default QoS config, don't start classifier. */
693         if (mrvl_qos_cfg  &&
694             mrvl_qos_cfg->port[dev->data->port_id].use_global_defaults == 0) {
695                 ret = mrvl_start_qos_mapping(priv);
696                 if (ret) {
697                         MRVL_LOG(ERR, "Failed to setup QoS mapping");
698                         goto out;
699                 }
700         }
701
702         ret = mrvl_dev_set_link_up(dev);
703         if (ret) {
704                 MRVL_LOG(ERR, "Failed to set link up");
705                 goto out;
706         }
707
708         /* start tx queues */
709         for (i = 0; i < dev->data->nb_tx_queues; i++) {
710                 struct mrvl_txq *txq = dev->data->tx_queues[i];
711
712                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
713
714                 if (!txq->tx_deferred_start)
715                         continue;
716
717                 /*
718                  * All txqs are started by default. Stop them
719                  * so that tx_deferred_start works as expected.
720                  */
721                 ret = mrvl_tx_queue_stop(dev, i);
722                 if (ret)
723                         goto out;
724         }
725
726         mrvl_flow_init(dev);
727         mrvl_mtr_init(dev);
728         mrvl_set_tx_function(dev);
729
730         return 0;
731 out:
732         MRVL_LOG(ERR, "Failed to start device");
733         pp2_ppio_deinit(priv->ppio);
734         return ret;
735 }
736
737 /**
738  * Flush receive queues.
739  *
740  * @param dev
741  *   Pointer to Ethernet device structure.
742  */
743 static void
744 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
745 {
746         int i;
747
748         MRVL_LOG(INFO, "Flushing rx queues");
749         for (i = 0; i < dev->data->nb_rx_queues; i++) {
750                 int ret, num;
751
752                 do {
753                         struct mrvl_rxq *q = dev->data->rx_queues[i];
754                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
755
756                         num = MRVL_PP2_RXD_MAX;
757                         ret = pp2_ppio_recv(q->priv->ppio,
758                                             q->priv->rxq_map[q->queue_id].tc,
759                                             q->priv->rxq_map[q->queue_id].inq,
760                                             descs, (uint16_t *)&num);
761                 } while (ret == 0 && num);
762         }
763 }
764
765 /**
766  * Flush transmit shadow queues.
767  *
768  * @param dev
769  *   Pointer to Ethernet device structure.
770  */
771 static void
772 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
773 {
774         int i, j;
775         struct mrvl_txq *txq;
776
777         MRVL_LOG(INFO, "Flushing tx shadow queues");
778         for (i = 0; i < dev->data->nb_tx_queues; i++) {
779                 txq = (struct mrvl_txq *)dev->data->tx_queues[i];
780
781                 for (j = 0; j < RTE_MAX_LCORE; j++) {
782                         struct mrvl_shadow_txq *sq;
783
784                         if (!hifs[j])
785                                 continue;
786
787                         sq = &txq->shadow_txqs[j];
788                         mrvl_free_sent_buffers(txq->priv->ppio,
789                                 hifs[j], j, sq, txq->queue_id, 1);
790                         while (sq->tail != sq->head) {
791                                 uint64_t addr = cookie_addr_high |
792                                         sq->ent[sq->tail].buff.cookie;
793                                 rte_pktmbuf_free(
794                                         (struct rte_mbuf *)addr);
795                                 sq->tail = (sq->tail + 1) &
796                                             MRVL_PP2_TX_SHADOWQ_MASK;
797                         }
798                         memset(sq, 0, sizeof(*sq));
799                 }
800         }
801 }
802
803 /**
804  * Flush hardware bpool (buffer-pool).
805  *
806  * @param dev
807  *   Pointer to Ethernet device structure.
808  */
809 static void
810 mrvl_flush_bpool(struct rte_eth_dev *dev)
811 {
812         struct mrvl_priv *priv = dev->data->dev_private;
813         struct pp2_hif *hif;
814         uint32_t num;
815         int ret;
816         unsigned int core_id = rte_lcore_id();
817
818         if (core_id == LCORE_ID_ANY)
819                 core_id = 0;
820
821         hif = mrvl_get_hif(priv, core_id);
822
823         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
824         if (ret) {
825                 MRVL_LOG(ERR, "Failed to get bpool buffers number");
826                 return;
827         }
828
829         while (num--) {
830                 struct pp2_buff_inf inf;
831                 uint64_t addr;
832
833                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
834                 if (ret)
835                         break;
836
837                 addr = cookie_addr_high | inf.cookie;
838                 rte_pktmbuf_free((struct rte_mbuf *)addr);
839         }
840 }
841
842 /**
843  * DPDK callback to stop the device.
844  *
845  * @param dev
846  *   Pointer to Ethernet device structure.
847  */
848 static void
849 mrvl_dev_stop(struct rte_eth_dev *dev)
850 {
851         mrvl_dev_set_link_down(dev);
852 }
853
854 /**
855  * DPDK callback to close the device.
856  *
857  * @param dev
858  *   Pointer to Ethernet device structure.
859  */
860 static void
861 mrvl_dev_close(struct rte_eth_dev *dev)
862 {
863         struct mrvl_priv *priv = dev->data->dev_private;
864         size_t i;
865
866         mrvl_flush_rx_queues(dev);
867         mrvl_flush_tx_shadow_queues(dev);
868         mrvl_flow_deinit(dev);
869         mrvl_mtr_deinit(dev);
870
871         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
872                 struct pp2_ppio_tc_params *tc_params =
873                         &priv->ppio_params.inqs_params.tcs_params[i];
874
875                 if (tc_params->inqs_params) {
876                         rte_free(tc_params->inqs_params);
877                         tc_params->inqs_params = NULL;
878                 }
879         }
880
881         if (priv->cls_tbl) {
882                 pp2_cls_tbl_deinit(priv->cls_tbl);
883                 priv->cls_tbl = NULL;
884         }
885
886         if (priv->qos_tbl) {
887                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
888                 priv->qos_tbl = NULL;
889         }
890
891         mrvl_flush_bpool(dev);
892         mrvl_tm_deinit(dev);
893
894         if (priv->ppio) {
895                 pp2_ppio_deinit(priv->ppio);
896                 priv->ppio = NULL;
897         }
898
899         /* policer must be released after ppio deinitialization */
900         if (priv->default_policer) {
901                 pp2_cls_plcr_deinit(priv->default_policer);
902                 priv->default_policer = NULL;
903         }
904
905
906         if (priv->bpool) {
907                 pp2_bpool_deinit(priv->bpool);
908                 used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
909                 priv->bpool = NULL;
910         }
911
912         mrvl_dev_num--;
913
914         if (mrvl_dev_num == 0) {
915                 MRVL_LOG(INFO, "Perform MUSDK deinit");
916                 mrvl_deinit_hifs();
917                 mrvl_deinit_pp2();
918                 rte_mvep_deinit(MVEP_MOD_T_PP2);
919         }
920 }
921
922 /**
923  * DPDK callback to retrieve physical link information.
924  *
925  * @param dev
926  *   Pointer to Ethernet device structure.
927  * @param wait_to_complete
928  *   Wait for request completion (ignored).
929  *
930  * @return
931  *   0 on success, negative error value otherwise.
932  */
933 static int
934 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
935 {
936         /*
937          * TODO
938          * once MUSDK provides necessary API use it here
939          */
940         struct mrvl_priv *priv = dev->data->dev_private;
941         struct ethtool_cmd edata;
942         struct ifreq req;
943         int ret, fd, link_up;
944
945         if (!priv->ppio)
946                 return -EPERM;
947
948         edata.cmd = ETHTOOL_GSET;
949
950         strcpy(req.ifr_name, dev->data->name);
951         req.ifr_data = (void *)&edata;
952
953         fd = socket(AF_INET, SOCK_DGRAM, 0);
954         if (fd == -1)
955                 return -EFAULT;
956
957         ret = ioctl(fd, SIOCETHTOOL, &req);
958         if (ret == -1) {
959                 close(fd);
960                 return -EFAULT;
961         }
962
963         close(fd);
964
965         switch (ethtool_cmd_speed(&edata)) {
966         case SPEED_10:
967                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
968                 break;
969         case SPEED_100:
970                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
971                 break;
972         case SPEED_1000:
973                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
974                 break;
975         case SPEED_10000:
976                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
977                 break;
978         default:
979                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
980         }
981
982         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
983                                                          ETH_LINK_HALF_DUPLEX;
984         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
985                                                            ETH_LINK_FIXED;
986         pp2_ppio_get_link_state(priv->ppio, &link_up);
987         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
988
989         return 0;
990 }
991
992 /**
993  * DPDK callback to enable promiscuous mode.
994  *
995  * @param dev
996  *   Pointer to Ethernet device structure.
997  *
998  * @return
999  *   0 on success, negative error value otherwise.
1000  */
1001 static int
1002 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
1003 {
1004         struct mrvl_priv *priv = dev->data->dev_private;
1005         int ret;
1006
1007         if (!priv->ppio)
1008                 return 0;
1009
1010         if (priv->isolated)
1011                 return 0;
1012
1013         ret = pp2_ppio_set_promisc(priv->ppio, 1);
1014         if (ret) {
1015                 MRVL_LOG(ERR, "Failed to enable promiscuous mode");
1016                 return -EAGAIN;
1017         }
1018
1019         return 0;
1020 }
1021
1022 /**
1023  * DPDK callback to enable allmulti mode.
1024  *
1025  * @param dev
1026  *   Pointer to Ethernet device structure.
1027  */
1028 static void
1029 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
1030 {
1031         struct mrvl_priv *priv = dev->data->dev_private;
1032         int ret;
1033
1034         if (!priv->ppio)
1035                 return;
1036
1037         if (priv->isolated)
1038                 return;
1039
1040         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
1041         if (ret)
1042                 MRVL_LOG(ERR, "Failed enable all-multicast mode");
1043 }
1044
1045 /**
1046  * DPDK callback to disable promiscuous mode.
1047  *
1048  * @param dev
1049  *   Pointer to Ethernet device structure.
1050  *
1051  * @return
1052  *   0 on success, negative error value otherwise.
1053  */
1054 static int
1055 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
1056 {
1057         struct mrvl_priv *priv = dev->data->dev_private;
1058         int ret;
1059
1060         if (!priv->ppio)
1061                 return 0;
1062
1063         ret = pp2_ppio_set_promisc(priv->ppio, 0);
1064         if (ret) {
1065                 MRVL_LOG(ERR, "Failed to disable promiscuous mode");
1066                 return -EAGAIN;
1067         }
1068
1069         return 0;
1070 }
1071
1072 /**
1073  * DPDK callback to disable allmulticast mode.
1074  *
1075  * @param dev
1076  *   Pointer to Ethernet device structure.
1077  */
1078 static void
1079 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
1080 {
1081         struct mrvl_priv *priv = dev->data->dev_private;
1082         int ret;
1083
1084         if (!priv->ppio)
1085                 return;
1086
1087         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
1088         if (ret)
1089                 MRVL_LOG(ERR, "Failed to disable all-multicast mode");
1090 }
1091
1092 /**
1093  * DPDK callback to remove a MAC address.
1094  *
1095  * @param dev
1096  *   Pointer to Ethernet device structure.
1097  * @param index
1098  *   MAC address index.
1099  */
1100 static void
1101 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1102 {
1103         struct mrvl_priv *priv = dev->data->dev_private;
1104         char buf[RTE_ETHER_ADDR_FMT_SIZE];
1105         int ret;
1106
1107         if (!priv->ppio)
1108                 return;
1109
1110         if (priv->isolated)
1111                 return;
1112
1113         ret = pp2_ppio_remove_mac_addr(priv->ppio,
1114                                        dev->data->mac_addrs[index].addr_bytes);
1115         if (ret) {
1116                 rte_ether_format_addr(buf, sizeof(buf),
1117                                   &dev->data->mac_addrs[index]);
1118                 MRVL_LOG(ERR, "Failed to remove mac %s", buf);
1119         }
1120 }
1121
1122 /**
1123  * DPDK callback to add a MAC address.
1124  *
1125  * @param dev
1126  *   Pointer to Ethernet device structure.
1127  * @param mac_addr
1128  *   MAC address to register.
1129  * @param index
1130  *   MAC address index.
1131  * @param vmdq
1132  *   VMDq pool index to associate address with (unused).
1133  *
1134  * @return
1135  *   0 on success, negative error value otherwise.
1136  */
1137 static int
1138 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1139                   uint32_t index, uint32_t vmdq __rte_unused)
1140 {
1141         struct mrvl_priv *priv = dev->data->dev_private;
1142         char buf[RTE_ETHER_ADDR_FMT_SIZE];
1143         int ret;
1144
1145         if (priv->isolated)
1146                 return -ENOTSUP;
1147
1148         if (index == 0)
1149                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
1150                 return -1;
1151
1152         if (!priv->ppio)
1153                 return 0;
1154
1155         /*
1156          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
1157          * parameter uc_filter_max. Maximum number of mc addresses is then
1158          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
1159          * 21 respectively.
1160          *
1161          * If more than uc_filter_max uc addresses were added to filter list
1162          * then NIC will switch to promiscuous mode automatically.
1163          *
1164          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
1165          * were added to filter list then NIC will switch to all-multicast mode
1166          * automatically.
1167          */
1168         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
1169         if (ret) {
1170                 rte_ether_format_addr(buf, sizeof(buf), mac_addr);
1171                 MRVL_LOG(ERR, "Failed to add mac %s", buf);
1172                 return -1;
1173         }
1174
1175         return 0;
1176 }
1177
1178 /**
1179  * DPDK callback to set the primary MAC address.
1180  *
1181  * @param dev
1182  *   Pointer to Ethernet device structure.
1183  * @param mac_addr
1184  *   MAC address to register.
1185  *
1186  * @return
1187  *   0 on success, negative error value otherwise.
1188  */
1189 static int
1190 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
1191 {
1192         struct mrvl_priv *priv = dev->data->dev_private;
1193         int ret;
1194
1195         if (!priv->ppio)
1196                 return 0;
1197
1198         if (priv->isolated)
1199                 return -ENOTSUP;
1200
1201         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
1202         if (ret) {
1203                 char buf[RTE_ETHER_ADDR_FMT_SIZE];
1204                 rte_ether_format_addr(buf, sizeof(buf), mac_addr);
1205                 MRVL_LOG(ERR, "Failed to set mac to %s", buf);
1206         }
1207
1208         return ret;
1209 }
1210
1211 /**
1212  * DPDK callback to get device statistics.
1213  *
1214  * @param dev
1215  *   Pointer to Ethernet device structure.
1216  * @param stats
1217  *   Stats structure output buffer.
1218  *
1219  * @return
1220  *   0 on success, negative error value otherwise.
1221  */
1222 static int
1223 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1224 {
1225         struct mrvl_priv *priv = dev->data->dev_private;
1226         struct pp2_ppio_statistics ppio_stats;
1227         uint64_t drop_mac = 0;
1228         unsigned int i, idx, ret;
1229
1230         if (!priv->ppio)
1231                 return -EPERM;
1232
1233         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1234                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1235                 struct pp2_ppio_inq_statistics rx_stats;
1236
1237                 if (!rxq)
1238                         continue;
1239
1240                 idx = rxq->queue_id;
1241                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1242                         MRVL_LOG(ERR,
1243                                 "rx queue %d stats out of range (0 - %d)",
1244                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1245                         continue;
1246                 }
1247
1248                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
1249                                                   priv->rxq_map[idx].tc,
1250                                                   priv->rxq_map[idx].inq,
1251                                                   &rx_stats, 0);
1252                 if (unlikely(ret)) {
1253                         MRVL_LOG(ERR,
1254                                 "Failed to update rx queue %d stats", idx);
1255                         break;
1256                 }
1257
1258                 stats->q_ibytes[idx] = rxq->bytes_recv;
1259                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
1260                 stats->q_errors[idx] = rx_stats.drop_early +
1261                                        rx_stats.drop_fullq +
1262                                        rx_stats.drop_bm +
1263                                        rxq->drop_mac;
1264                 stats->ibytes += rxq->bytes_recv;
1265                 drop_mac += rxq->drop_mac;
1266         }
1267
1268         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1269                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1270                 struct pp2_ppio_outq_statistics tx_stats;
1271
1272                 if (!txq)
1273                         continue;
1274
1275                 idx = txq->queue_id;
1276                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1277                         MRVL_LOG(ERR,
1278                                 "tx queue %d stats out of range (0 - %d)",
1279                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1280                 }
1281
1282                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1283                                                    &tx_stats, 0);
1284                 if (unlikely(ret)) {
1285                         MRVL_LOG(ERR,
1286                                 "Failed to update tx queue %d stats", idx);
1287                         break;
1288                 }
1289
1290                 stats->q_opackets[idx] = tx_stats.deq_desc;
1291                 stats->q_obytes[idx] = txq->bytes_sent;
1292                 stats->obytes += txq->bytes_sent;
1293         }
1294
1295         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1296         if (unlikely(ret)) {
1297                 MRVL_LOG(ERR, "Failed to update port statistics");
1298                 return ret;
1299         }
1300
1301         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1302         stats->opackets += ppio_stats.tx_packets;
1303         stats->imissed += ppio_stats.rx_fullq_dropped +
1304                           ppio_stats.rx_bm_dropped +
1305                           ppio_stats.rx_early_dropped +
1306                           ppio_stats.rx_fifo_dropped +
1307                           ppio_stats.rx_cls_dropped;
1308         stats->ierrors = drop_mac;
1309
1310         return 0;
1311 }
1312
1313 /**
1314  * DPDK callback to clear device statistics.
1315  *
1316  * @param dev
1317  *   Pointer to Ethernet device structure.
1318  *
1319  * @return
1320  *   0 on success, negative error value otherwise.
1321  */
1322 static int
1323 mrvl_stats_reset(struct rte_eth_dev *dev)
1324 {
1325         struct mrvl_priv *priv = dev->data->dev_private;
1326         int i;
1327
1328         if (!priv->ppio)
1329                 return 0;
1330
1331         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1332                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1333
1334                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1335                                             priv->rxq_map[i].inq, NULL, 1);
1336                 rxq->bytes_recv = 0;
1337                 rxq->drop_mac = 0;
1338         }
1339
1340         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1341                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1342
1343                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1344                 txq->bytes_sent = 0;
1345         }
1346
1347         return pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1348 }
1349
1350 /**
1351  * DPDK callback to get extended statistics.
1352  *
1353  * @param dev
1354  *   Pointer to Ethernet device structure.
1355  * @param stats
1356  *   Pointer to xstats table.
1357  * @param n
1358  *   Number of entries in xstats table.
1359  * @return
1360  *   Negative value on error, number of read xstats otherwise.
1361  */
1362 static int
1363 mrvl_xstats_get(struct rte_eth_dev *dev,
1364                 struct rte_eth_xstat *stats, unsigned int n)
1365 {
1366         struct mrvl_priv *priv = dev->data->dev_private;
1367         struct pp2_ppio_statistics ppio_stats;
1368         unsigned int i;
1369
1370         if (!stats)
1371                 return 0;
1372
1373         pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1374         for (i = 0; i < n && i < RTE_DIM(mrvl_xstats_tbl); i++) {
1375                 uint64_t val;
1376
1377                 if (mrvl_xstats_tbl[i].size == sizeof(uint32_t))
1378                         val = *(uint32_t *)((uint8_t *)&ppio_stats +
1379                                             mrvl_xstats_tbl[i].offset);
1380                 else if (mrvl_xstats_tbl[i].size == sizeof(uint64_t))
1381                         val = *(uint64_t *)((uint8_t *)&ppio_stats +
1382                                             mrvl_xstats_tbl[i].offset);
1383                 else
1384                         return -EINVAL;
1385
1386                 stats[i].id = i;
1387                 stats[i].value = val;
1388         }
1389
1390         return n;
1391 }
1392
1393 /**
1394  * DPDK callback to reset extended statistics.
1395  *
1396  * @param dev
1397  *   Pointer to Ethernet device structure.
1398  *
1399  * @return
1400  *   0 on success, negative error value otherwise.
1401  */
1402 static int
1403 mrvl_xstats_reset(struct rte_eth_dev *dev)
1404 {
1405         return mrvl_stats_reset(dev);
1406 }
1407
1408 /**
1409  * DPDK callback to get extended statistics names.
1410  *
1411  * @param dev (unused)
1412  *   Pointer to Ethernet device structure.
1413  * @param xstats_names
1414  *   Pointer to xstats names table.
1415  * @param size
1416  *   Size of the xstats names table.
1417  * @return
1418  *   Number of read names.
1419  */
1420 static int
1421 mrvl_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
1422                       struct rte_eth_xstat_name *xstats_names,
1423                       unsigned int size)
1424 {
1425         unsigned int i;
1426
1427         if (!xstats_names)
1428                 return RTE_DIM(mrvl_xstats_tbl);
1429
1430         for (i = 0; i < size && i < RTE_DIM(mrvl_xstats_tbl); i++)
1431                 strlcpy(xstats_names[i].name, mrvl_xstats_tbl[i].name,
1432                         RTE_ETH_XSTATS_NAME_SIZE);
1433
1434         return size;
1435 }
1436
1437 /**
1438  * DPDK callback to get information about the device.
1439  *
1440  * @param dev
1441  *   Pointer to Ethernet device structure (unused).
1442  * @param info
1443  *   Info structure output buffer.
1444  */
1445 static int
1446 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
1447                    struct rte_eth_dev_info *info)
1448 {
1449         info->speed_capa = ETH_LINK_SPEED_10M |
1450                            ETH_LINK_SPEED_100M |
1451                            ETH_LINK_SPEED_1G |
1452                            ETH_LINK_SPEED_10G;
1453
1454         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1455         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1456         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1457
1458         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1459         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1460         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1461
1462         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1463         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1464         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1465
1466         info->rx_offload_capa = MRVL_RX_OFFLOADS;
1467         info->rx_queue_offload_capa = MRVL_RX_OFFLOADS;
1468
1469         info->tx_offload_capa = MRVL_TX_OFFLOADS;
1470         info->tx_queue_offload_capa = MRVL_TX_OFFLOADS;
1471
1472         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1473                                        ETH_RSS_NONFRAG_IPV4_TCP |
1474                                        ETH_RSS_NONFRAG_IPV4_UDP;
1475
1476         /* By default packets are dropped if no descriptors are available */
1477         info->default_rxconf.rx_drop_en = 1;
1478
1479         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1480
1481         return 0;
1482 }
1483
1484 /**
1485  * Return supported packet types.
1486  *
1487  * @param dev
1488  *   Pointer to Ethernet device structure (unused).
1489  *
1490  * @return
1491  *   Const pointer to the table with supported packet types.
1492  */
1493 static const uint32_t *
1494 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1495 {
1496         static const uint32_t ptypes[] = {
1497                 RTE_PTYPE_L2_ETHER,
1498                 RTE_PTYPE_L2_ETHER_VLAN,
1499                 RTE_PTYPE_L2_ETHER_QINQ,
1500                 RTE_PTYPE_L3_IPV4,
1501                 RTE_PTYPE_L3_IPV4_EXT,
1502                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1503                 RTE_PTYPE_L3_IPV6,
1504                 RTE_PTYPE_L3_IPV6_EXT,
1505                 RTE_PTYPE_L2_ETHER_ARP,
1506                 RTE_PTYPE_L4_TCP,
1507                 RTE_PTYPE_L4_UDP
1508         };
1509
1510         return ptypes;
1511 }
1512
1513 /**
1514  * DPDK callback to get information about specific receive queue.
1515  *
1516  * @param dev
1517  *   Pointer to Ethernet device structure.
1518  * @param rx_queue_id
1519  *   Receive queue index.
1520  * @param qinfo
1521  *   Receive queue information structure.
1522  */
1523 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1524                               struct rte_eth_rxq_info *qinfo)
1525 {
1526         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1527         struct mrvl_priv *priv = dev->data->dev_private;
1528         int inq = priv->rxq_map[rx_queue_id].inq;
1529         int tc = priv->rxq_map[rx_queue_id].tc;
1530         struct pp2_ppio_tc_params *tc_params =
1531                 &priv->ppio_params.inqs_params.tcs_params[tc];
1532
1533         qinfo->mp = q->mp;
1534         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1535 }
1536
1537 /**
1538  * DPDK callback to get information about specific transmit queue.
1539  *
1540  * @param dev
1541  *   Pointer to Ethernet device structure.
1542  * @param tx_queue_id
1543  *   Transmit queue index.
1544  * @param qinfo
1545  *   Transmit queue information structure.
1546  */
1547 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1548                               struct rte_eth_txq_info *qinfo)
1549 {
1550         struct mrvl_priv *priv = dev->data->dev_private;
1551         struct mrvl_txq *txq = dev->data->tx_queues[tx_queue_id];
1552
1553         qinfo->nb_desc =
1554                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1555         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1556 }
1557
1558 /**
1559  * DPDK callback to Configure a VLAN filter.
1560  *
1561  * @param dev
1562  *   Pointer to Ethernet device structure.
1563  * @param vlan_id
1564  *   VLAN ID to filter.
1565  * @param on
1566  *   Toggle filter.
1567  *
1568  * @return
1569  *   0 on success, negative error value otherwise.
1570  */
1571 static int
1572 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1573 {
1574         struct mrvl_priv *priv = dev->data->dev_private;
1575
1576         if (!priv->ppio)
1577                 return -EPERM;
1578
1579         if (priv->isolated)
1580                 return -ENOTSUP;
1581
1582         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1583                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1584 }
1585
1586 /**
1587  * Release buffers to hardware bpool (buffer-pool)
1588  *
1589  * @param rxq
1590  *   Receive queue pointer.
1591  * @param num
1592  *   Number of buffers to release to bpool.
1593  *
1594  * @return
1595  *   0 on success, negative error value otherwise.
1596  */
1597 static int
1598 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1599 {
1600         struct buff_release_entry entries[MRVL_PP2_RXD_MAX];
1601         struct rte_mbuf *mbufs[MRVL_PP2_RXD_MAX];
1602         int i, ret;
1603         unsigned int core_id;
1604         struct pp2_hif *hif;
1605         struct pp2_bpool *bpool;
1606
1607         core_id = rte_lcore_id();
1608         if (core_id == LCORE_ID_ANY)
1609                 core_id = 0;
1610
1611         hif = mrvl_get_hif(rxq->priv, core_id);
1612         if (!hif)
1613                 return -1;
1614
1615         bpool = rxq->priv->bpool;
1616
1617         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1618         if (ret)
1619                 return ret;
1620
1621         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1622                 cookie_addr_high =
1623                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1624
1625         for (i = 0; i < num; i++) {
1626                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1627                         != cookie_addr_high) {
1628                         MRVL_LOG(ERR,
1629                                 "mbuf virtual addr high 0x%lx out of range",
1630                                 (uint64_t)mbufs[i] >> 32);
1631                         goto out;
1632                 }
1633
1634                 entries[i].buff.addr =
1635                         rte_mbuf_data_iova_default(mbufs[i]);
1636                 entries[i].buff.cookie = (uint64_t)mbufs[i];
1637                 entries[i].bpool = bpool;
1638         }
1639
1640         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1641         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1642
1643         if (i != num)
1644                 goto out;
1645
1646         return 0;
1647 out:
1648         for (; i < num; i++)
1649                 rte_pktmbuf_free(mbufs[i]);
1650
1651         return -1;
1652 }
1653
1654 /**
1655  * DPDK callback to configure the receive queue.
1656  *
1657  * @param dev
1658  *   Pointer to Ethernet device structure.
1659  * @param idx
1660  *   RX queue index.
1661  * @param desc
1662  *   Number of descriptors to configure in queue.
1663  * @param socket
1664  *   NUMA socket on which memory must be allocated.
1665  * @param conf
1666  *   Thresholds parameters.
1667  * @param mp
1668  *   Memory pool for buffer allocations.
1669  *
1670  * @return
1671  *   0 on success, negative error value otherwise.
1672  */
1673 static int
1674 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1675                     unsigned int socket,
1676                     const struct rte_eth_rxconf *conf,
1677                     struct rte_mempool *mp)
1678 {
1679         struct mrvl_priv *priv = dev->data->dev_private;
1680         struct mrvl_rxq *rxq;
1681         uint32_t frame_size, buf_size = rte_pktmbuf_data_room_size(mp);
1682         uint32_t max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1683         int ret, tc, inq;
1684         uint64_t offloads;
1685
1686         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1687
1688         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1689                 /*
1690                  * Unknown TC mapping, mapping will not have a correct queue.
1691                  */
1692                 MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu",
1693                         idx, priv->ppio_id);
1694                 return -EFAULT;
1695         }
1696
1697         frame_size = buf_size - RTE_PKTMBUF_HEADROOM - MRVL_PKT_EFFEC_OFFS;
1698         if (frame_size < max_rx_pkt_len) {
1699                 MRVL_LOG(WARNING,
1700                         "Mbuf size must be increased to %u bytes to hold up "
1701                         "to %u bytes of data.",
1702                         buf_size + max_rx_pkt_len - frame_size,
1703                         max_rx_pkt_len);
1704                 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
1705                 MRVL_LOG(INFO, "Setting max rx pkt len to %u",
1706                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
1707         }
1708
1709         if (dev->data->rx_queues[idx]) {
1710                 rte_free(dev->data->rx_queues[idx]);
1711                 dev->data->rx_queues[idx] = NULL;
1712         }
1713
1714         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1715         if (!rxq)
1716                 return -ENOMEM;
1717
1718         rxq->priv = priv;
1719         rxq->mp = mp;
1720         rxq->cksum_enabled = offloads & DEV_RX_OFFLOAD_IPV4_CKSUM;
1721         rxq->queue_id = idx;
1722         rxq->port_id = dev->data->port_id;
1723         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1724
1725         tc = priv->rxq_map[rxq->queue_id].tc,
1726         inq = priv->rxq_map[rxq->queue_id].inq;
1727         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1728                 desc;
1729
1730         ret = mrvl_fill_bpool(rxq, desc);
1731         if (ret) {
1732                 rte_free(rxq);
1733                 return ret;
1734         }
1735
1736         priv->bpool_init_size += desc;
1737
1738         dev->data->rx_queues[idx] = rxq;
1739
1740         return 0;
1741 }
1742
1743 /**
1744  * DPDK callback to release the receive queue.
1745  *
1746  * @param rxq
1747  *   Generic receive queue pointer.
1748  */
1749 static void
1750 mrvl_rx_queue_release(void *rxq)
1751 {
1752         struct mrvl_rxq *q = rxq;
1753         struct pp2_ppio_tc_params *tc_params;
1754         int i, num, tc, inq;
1755         struct pp2_hif *hif;
1756         unsigned int core_id = rte_lcore_id();
1757
1758         if (core_id == LCORE_ID_ANY)
1759                 core_id = 0;
1760
1761         if (!q)
1762                 return;
1763
1764         hif = mrvl_get_hif(q->priv, core_id);
1765
1766         if (!hif)
1767                 return;
1768
1769         tc = q->priv->rxq_map[q->queue_id].tc;
1770         inq = q->priv->rxq_map[q->queue_id].inq;
1771         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1772         num = tc_params->inqs_params[inq].size;
1773         for (i = 0; i < num; i++) {
1774                 struct pp2_buff_inf inf;
1775                 uint64_t addr;
1776
1777                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
1778                 addr = cookie_addr_high | inf.cookie;
1779                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1780         }
1781
1782         rte_free(q);
1783 }
1784
1785 /**
1786  * DPDK callback to configure the transmit queue.
1787  *
1788  * @param dev
1789  *   Pointer to Ethernet device structure.
1790  * @param idx
1791  *   Transmit queue index.
1792  * @param desc
1793  *   Number of descriptors to configure in the queue.
1794  * @param socket
1795  *   NUMA socket on which memory must be allocated.
1796  * @param conf
1797  *   Tx queue configuration parameters.
1798  *
1799  * @return
1800  *   0 on success, negative error value otherwise.
1801  */
1802 static int
1803 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1804                     unsigned int socket,
1805                     const struct rte_eth_txconf *conf)
1806 {
1807         struct mrvl_priv *priv = dev->data->dev_private;
1808         struct mrvl_txq *txq;
1809
1810         if (dev->data->tx_queues[idx]) {
1811                 rte_free(dev->data->tx_queues[idx]);
1812                 dev->data->tx_queues[idx] = NULL;
1813         }
1814
1815         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1816         if (!txq)
1817                 return -ENOMEM;
1818
1819         txq->priv = priv;
1820         txq->queue_id = idx;
1821         txq->port_id = dev->data->port_id;
1822         txq->tx_deferred_start = conf->tx_deferred_start;
1823         dev->data->tx_queues[idx] = txq;
1824
1825         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1826
1827         return 0;
1828 }
1829
1830 /**
1831  * DPDK callback to release the transmit queue.
1832  *
1833  * @param txq
1834  *   Generic transmit queue pointer.
1835  */
1836 static void
1837 mrvl_tx_queue_release(void *txq)
1838 {
1839         struct mrvl_txq *q = txq;
1840
1841         if (!q)
1842                 return;
1843
1844         rte_free(q);
1845 }
1846
1847 /**
1848  * DPDK callback to get flow control configuration.
1849  *
1850  * @param dev
1851  *  Pointer to Ethernet device structure.
1852  * @param fc_conf
1853  *  Pointer to the flow control configuration.
1854  *
1855  * @return
1856  *  0 on success, negative error value otherwise.
1857  */
1858 static int
1859 mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1860 {
1861         struct mrvl_priv *priv = dev->data->dev_private;
1862         int ret, en;
1863
1864         if (!priv)
1865                 return -EPERM;
1866
1867         ret = pp2_ppio_get_rx_pause(priv->ppio, &en);
1868         if (ret) {
1869                 MRVL_LOG(ERR, "Failed to read rx pause state");
1870                 return ret;
1871         }
1872
1873         fc_conf->mode = en ? RTE_FC_RX_PAUSE : RTE_FC_NONE;
1874
1875         return 0;
1876 }
1877
1878 /**
1879  * DPDK callback to set flow control configuration.
1880  *
1881  * @param dev
1882  *  Pointer to Ethernet device structure.
1883  * @param fc_conf
1884  *  Pointer to the flow control configuration.
1885  *
1886  * @return
1887  *  0 on success, negative error value otherwise.
1888  */
1889 static int
1890 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1891 {
1892         struct mrvl_priv *priv = dev->data->dev_private;
1893
1894         if (!priv)
1895                 return -EPERM;
1896
1897         if (fc_conf->high_water ||
1898             fc_conf->low_water ||
1899             fc_conf->pause_time ||
1900             fc_conf->mac_ctrl_frame_fwd ||
1901             fc_conf->autoneg) {
1902                 MRVL_LOG(ERR, "Flowctrl parameter is not supported");
1903
1904                 return -EINVAL;
1905         }
1906
1907         if (fc_conf->mode == RTE_FC_NONE ||
1908             fc_conf->mode == RTE_FC_RX_PAUSE) {
1909                 int ret, en;
1910
1911                 en = fc_conf->mode == RTE_FC_NONE ? 0 : 1;
1912                 ret = pp2_ppio_set_rx_pause(priv->ppio, en);
1913                 if (ret)
1914                         MRVL_LOG(ERR,
1915                                 "Failed to change flowctrl on RX side");
1916
1917                 return ret;
1918         }
1919
1920         return 0;
1921 }
1922
1923 /**
1924  * Update RSS hash configuration
1925  *
1926  * @param dev
1927  *   Pointer to Ethernet device structure.
1928  * @param rss_conf
1929  *   Pointer to RSS configuration.
1930  *
1931  * @return
1932  *   0 on success, negative error value otherwise.
1933  */
1934 static int
1935 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1936                      struct rte_eth_rss_conf *rss_conf)
1937 {
1938         struct mrvl_priv *priv = dev->data->dev_private;
1939
1940         if (priv->isolated)
1941                 return -ENOTSUP;
1942
1943         return mrvl_configure_rss(priv, rss_conf);
1944 }
1945
1946 /**
1947  * DPDK callback to get RSS hash configuration.
1948  *
1949  * @param dev
1950  *   Pointer to Ethernet device structure.
1951  * @rss_conf
1952  *   Pointer to RSS configuration.
1953  *
1954  * @return
1955  *   Always 0.
1956  */
1957 static int
1958 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1959                        struct rte_eth_rss_conf *rss_conf)
1960 {
1961         struct mrvl_priv *priv = dev->data->dev_private;
1962         enum pp2_ppio_hash_type hash_type =
1963                 priv->ppio_params.inqs_params.hash_type;
1964
1965         rss_conf->rss_key = NULL;
1966
1967         if (hash_type == PP2_PPIO_HASH_T_NONE)
1968                 rss_conf->rss_hf = 0;
1969         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1970                 rss_conf->rss_hf = ETH_RSS_IPV4;
1971         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1972                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1973         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1974                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1975
1976         return 0;
1977 }
1978
1979 /**
1980  * DPDK callback to get rte_flow callbacks.
1981  *
1982  * @param dev
1983  *   Pointer to the device structure.
1984  * @param filer_type
1985  *   Flow filter type.
1986  * @param filter_op
1987  *   Flow filter operation.
1988  * @param arg
1989  *   Pointer to pass the flow ops.
1990  *
1991  * @return
1992  *   0 on success, negative error value otherwise.
1993  */
1994 static int
1995 mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
1996                      enum rte_filter_type filter_type,
1997                      enum rte_filter_op filter_op, void *arg)
1998 {
1999         switch (filter_type) {
2000         case RTE_ETH_FILTER_GENERIC:
2001                 if (filter_op != RTE_ETH_FILTER_GET)
2002                         return -EINVAL;
2003                 *(const void **)arg = &mrvl_flow_ops;
2004                 return 0;
2005         default:
2006                 MRVL_LOG(WARNING, "Filter type (%d) not supported",
2007                                 filter_type);
2008                 return -EINVAL;
2009         }
2010 }
2011
2012 /**
2013  * DPDK callback to get rte_mtr callbacks.
2014  *
2015  * @param dev
2016  *   Pointer to the device structure.
2017  * @param ops
2018  *   Pointer to pass the mtr ops.
2019  *
2020  * @return
2021  *   Always 0.
2022  */
2023 static int
2024 mrvl_mtr_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
2025 {
2026         *(const void **)ops = &mrvl_mtr_ops;
2027
2028         return 0;
2029 }
2030
2031 /**
2032  * DPDK callback to get rte_tm callbacks.
2033  *
2034  * @param dev
2035  *   Pointer to the device structure.
2036  * @param ops
2037  *   Pointer to pass the tm ops.
2038  *
2039  * @return
2040  *   Always 0.
2041  */
2042 static int
2043 mrvl_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
2044 {
2045         *(const void **)ops = &mrvl_tm_ops;
2046
2047         return 0;
2048 }
2049
2050 static const struct eth_dev_ops mrvl_ops = {
2051         .dev_configure = mrvl_dev_configure,
2052         .dev_start = mrvl_dev_start,
2053         .dev_stop = mrvl_dev_stop,
2054         .dev_set_link_up = mrvl_dev_set_link_up,
2055         .dev_set_link_down = mrvl_dev_set_link_down,
2056         .dev_close = mrvl_dev_close,
2057         .link_update = mrvl_link_update,
2058         .promiscuous_enable = mrvl_promiscuous_enable,
2059         .allmulticast_enable = mrvl_allmulticast_enable,
2060         .promiscuous_disable = mrvl_promiscuous_disable,
2061         .allmulticast_disable = mrvl_allmulticast_disable,
2062         .mac_addr_remove = mrvl_mac_addr_remove,
2063         .mac_addr_add = mrvl_mac_addr_add,
2064         .mac_addr_set = mrvl_mac_addr_set,
2065         .mtu_set = mrvl_mtu_set,
2066         .stats_get = mrvl_stats_get,
2067         .stats_reset = mrvl_stats_reset,
2068         .xstats_get = mrvl_xstats_get,
2069         .xstats_reset = mrvl_xstats_reset,
2070         .xstats_get_names = mrvl_xstats_get_names,
2071         .dev_infos_get = mrvl_dev_infos_get,
2072         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
2073         .rxq_info_get = mrvl_rxq_info_get,
2074         .txq_info_get = mrvl_txq_info_get,
2075         .vlan_filter_set = mrvl_vlan_filter_set,
2076         .tx_queue_start = mrvl_tx_queue_start,
2077         .tx_queue_stop = mrvl_tx_queue_stop,
2078         .rx_queue_setup = mrvl_rx_queue_setup,
2079         .rx_queue_release = mrvl_rx_queue_release,
2080         .tx_queue_setup = mrvl_tx_queue_setup,
2081         .tx_queue_release = mrvl_tx_queue_release,
2082         .flow_ctrl_get = mrvl_flow_ctrl_get,
2083         .flow_ctrl_set = mrvl_flow_ctrl_set,
2084         .rss_hash_update = mrvl_rss_hash_update,
2085         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
2086         .filter_ctrl = mrvl_eth_filter_ctrl,
2087         .mtr_ops_get = mrvl_mtr_ops_get,
2088         .tm_ops_get = mrvl_tm_ops_get,
2089 };
2090
2091 /**
2092  * Return packet type information and l3/l4 offsets.
2093  *
2094  * @param desc
2095  *   Pointer to the received packet descriptor.
2096  * @param l3_offset
2097  *   l3 packet offset.
2098  * @param l4_offset
2099  *   l4 packet offset.
2100  *
2101  * @return
2102  *   Packet type information.
2103  */
2104 static inline uint64_t
2105 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
2106                                     uint8_t *l3_offset, uint8_t *l4_offset)
2107 {
2108         enum pp2_inq_l3_type l3_type;
2109         enum pp2_inq_l4_type l4_type;
2110         enum pp2_inq_vlan_tag vlan_tag;
2111         uint64_t packet_type;
2112
2113         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
2114         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
2115         pp2_ppio_inq_desc_get_vlan_tag(desc, &vlan_tag);
2116
2117         packet_type = RTE_PTYPE_L2_ETHER;
2118
2119         switch (vlan_tag) {
2120         case PP2_INQ_VLAN_TAG_SINGLE:
2121                 packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
2122                 break;
2123         case PP2_INQ_VLAN_TAG_DOUBLE:
2124         case PP2_INQ_VLAN_TAG_TRIPLE:
2125                 packet_type |= RTE_PTYPE_L2_ETHER_QINQ;
2126                 break;
2127         default:
2128                 break;
2129         }
2130
2131         switch (l3_type) {
2132         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
2133                 packet_type |= RTE_PTYPE_L3_IPV4;
2134                 break;
2135         case PP2_INQ_L3_TYPE_IPV4_OK:
2136                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
2137                 break;
2138         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
2139                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
2140                 break;
2141         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
2142                 packet_type |= RTE_PTYPE_L3_IPV6;
2143                 break;
2144         case PP2_INQ_L3_TYPE_IPV6_EXT:
2145                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
2146                 break;
2147         case PP2_INQ_L3_TYPE_ARP:
2148                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
2149                 /*
2150                  * In case of ARP l4_offset is set to wrong value.
2151                  * Set it to proper one so that later on mbuf->l3_len can be
2152                  * calculated subtracting l4_offset and l3_offset.
2153                  */
2154                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
2155                 break;
2156         default:
2157                 MRVL_LOG(DEBUG, "Failed to recognise l3 packet type");
2158                 break;
2159         }
2160
2161         switch (l4_type) {
2162         case PP2_INQ_L4_TYPE_TCP:
2163                 packet_type |= RTE_PTYPE_L4_TCP;
2164                 break;
2165         case PP2_INQ_L4_TYPE_UDP:
2166                 packet_type |= RTE_PTYPE_L4_UDP;
2167                 break;
2168         default:
2169                 MRVL_LOG(DEBUG, "Failed to recognise l4 packet type");
2170                 break;
2171         }
2172
2173         return packet_type;
2174 }
2175
2176 /**
2177  * Get offload information from the received packet descriptor.
2178  *
2179  * @param desc
2180  *   Pointer to the received packet descriptor.
2181  *
2182  * @return
2183  *   Mbuf offload flags.
2184  */
2185 static inline uint64_t
2186 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
2187 {
2188         uint64_t flags;
2189         enum pp2_inq_desc_status status;
2190
2191         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
2192         if (unlikely(status != PP2_DESC_ERR_OK))
2193                 flags = PKT_RX_IP_CKSUM_BAD;
2194         else
2195                 flags = PKT_RX_IP_CKSUM_GOOD;
2196
2197         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
2198         if (unlikely(status != PP2_DESC_ERR_OK))
2199                 flags |= PKT_RX_L4_CKSUM_BAD;
2200         else
2201                 flags |= PKT_RX_L4_CKSUM_GOOD;
2202
2203         return flags;
2204 }
2205
2206 /**
2207  * DPDK callback for receive.
2208  *
2209  * @param rxq
2210  *   Generic pointer to the receive queue.
2211  * @param rx_pkts
2212  *   Array to store received packets.
2213  * @param nb_pkts
2214  *   Maximum number of packets in array.
2215  *
2216  * @return
2217  *   Number of packets successfully received.
2218  */
2219 static uint16_t
2220 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2221 {
2222         struct mrvl_rxq *q = rxq;
2223         struct pp2_ppio_desc descs[nb_pkts];
2224         struct pp2_bpool *bpool;
2225         int i, ret, rx_done = 0;
2226         int num;
2227         struct pp2_hif *hif;
2228         unsigned int core_id = rte_lcore_id();
2229
2230         hif = mrvl_get_hif(q->priv, core_id);
2231
2232         if (unlikely(!q->priv->ppio || !hif))
2233                 return 0;
2234
2235         bpool = q->priv->bpool;
2236
2237         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
2238                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
2239         if (unlikely(ret < 0)) {
2240                 MRVL_LOG(ERR, "Failed to receive packets");
2241                 return 0;
2242         }
2243         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
2244
2245         for (i = 0; i < nb_pkts; i++) {
2246                 struct rte_mbuf *mbuf;
2247                 uint8_t l3_offset, l4_offset;
2248                 enum pp2_inq_desc_status status;
2249                 uint64_t addr;
2250
2251                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2252                         struct pp2_ppio_desc *pref_desc;
2253                         u64 pref_addr;
2254
2255                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
2256                         pref_addr = cookie_addr_high |
2257                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
2258                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
2259                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
2260                 }
2261
2262                 addr = cookie_addr_high |
2263                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
2264                 mbuf = (struct rte_mbuf *)addr;
2265                 rte_pktmbuf_reset(mbuf);
2266
2267                 /* drop packet in case of mac, overrun or resource error */
2268                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
2269                 if (unlikely(status != PP2_DESC_ERR_OK)) {
2270                         struct pp2_buff_inf binf = {
2271                                 .addr = rte_mbuf_data_iova_default(mbuf),
2272                                 .cookie = (uint64_t)mbuf,
2273                         };
2274
2275                         pp2_bpool_put_buff(hif, bpool, &binf);
2276                         mrvl_port_bpool_size
2277                                 [bpool->pp2_id][bpool->id][core_id]++;
2278                         q->drop_mac++;
2279                         continue;
2280                 }
2281
2282                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
2283                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
2284                 mbuf->data_len = mbuf->pkt_len;
2285                 mbuf->port = q->port_id;
2286                 mbuf->packet_type =
2287                         mrvl_desc_to_packet_type_and_offset(&descs[i],
2288                                                             &l3_offset,
2289                                                             &l4_offset);
2290                 mbuf->l2_len = l3_offset;
2291                 mbuf->l3_len = l4_offset - l3_offset;
2292
2293                 if (likely(q->cksum_enabled))
2294                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
2295
2296                 rx_pkts[rx_done++] = mbuf;
2297                 q->bytes_recv += mbuf->pkt_len;
2298         }
2299
2300         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
2301                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
2302
2303                 if (unlikely(num <= q->priv->bpool_min_size ||
2304                              (!rx_done && num < q->priv->bpool_init_size))) {
2305                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
2306                         if (ret)
2307                                 MRVL_LOG(ERR, "Failed to fill bpool");
2308                 } else if (unlikely(num > q->priv->bpool_max_size)) {
2309                         int i;
2310                         int pkt_to_remove = num - q->priv->bpool_init_size;
2311                         struct rte_mbuf *mbuf;
2312                         struct pp2_buff_inf buff;
2313
2314                         MRVL_LOG(DEBUG,
2315                                 "port-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)",
2316                                 bpool->pp2_id, q->priv->ppio->port_id,
2317                                 bpool->id, pkt_to_remove, num,
2318                                 q->priv->bpool_init_size);
2319
2320                         for (i = 0; i < pkt_to_remove; i++) {
2321                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
2322                                 if (ret)
2323                                         break;
2324                                 mbuf = (struct rte_mbuf *)
2325                                         (cookie_addr_high | buff.cookie);
2326                                 rte_pktmbuf_free(mbuf);
2327                         }
2328                         mrvl_port_bpool_size
2329                                 [bpool->pp2_id][bpool->id][core_id] -= i;
2330                 }
2331                 rte_spinlock_unlock(&q->priv->lock);
2332         }
2333
2334         return rx_done;
2335 }
2336
2337 /**
2338  * Prepare offload information.
2339  *
2340  * @param ol_flags
2341  *   Offload flags.
2342  * @param packet_type
2343  *   Packet type bitfield.
2344  * @param l3_type
2345  *   Pointer to the pp2_ouq_l3_type structure.
2346  * @param l4_type
2347  *   Pointer to the pp2_outq_l4_type structure.
2348  * @param gen_l3_cksum
2349  *   Will be set to 1 in case l3 checksum is computed.
2350  * @param l4_cksum
2351  *   Will be set to 1 in case l4 checksum is computed.
2352  *
2353  * @return
2354  *   0 on success, negative error value otherwise.
2355  */
2356 static inline int
2357 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
2358                         enum pp2_outq_l3_type *l3_type,
2359                         enum pp2_outq_l4_type *l4_type,
2360                         int *gen_l3_cksum,
2361                         int *gen_l4_cksum)
2362 {
2363         /*
2364          * Based on ol_flags prepare information
2365          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
2366          * for offloading.
2367          */
2368         if (ol_flags & PKT_TX_IPV4) {
2369                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
2370                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
2371         } else if (ol_flags & PKT_TX_IPV6) {
2372                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
2373                 /* no checksum for ipv6 header */
2374                 *gen_l3_cksum = 0;
2375         } else {
2376                 /* if something different then stop processing */
2377                 return -1;
2378         }
2379
2380         ol_flags &= PKT_TX_L4_MASK;
2381         if ((packet_type & RTE_PTYPE_L4_TCP) &&
2382             ol_flags == PKT_TX_TCP_CKSUM) {
2383                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
2384                 *gen_l4_cksum = 1;
2385         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
2386                    ol_flags == PKT_TX_UDP_CKSUM) {
2387                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
2388                 *gen_l4_cksum = 1;
2389         } else {
2390                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
2391                 /* no checksum for other type */
2392                 *gen_l4_cksum = 0;
2393         }
2394
2395         return 0;
2396 }
2397
2398 /**
2399  * Release already sent buffers to bpool (buffer-pool).
2400  *
2401  * @param ppio
2402  *   Pointer to the port structure.
2403  * @param hif
2404  *   Pointer to the MUSDK hardware interface.
2405  * @param sq
2406  *   Pointer to the shadow queue.
2407  * @param qid
2408  *   Queue id number.
2409  * @param force
2410  *   Force releasing packets.
2411  */
2412 static inline void
2413 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
2414                        unsigned int core_id, struct mrvl_shadow_txq *sq,
2415                        int qid, int force)
2416 {
2417         struct buff_release_entry *entry;
2418         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
2419         int i;
2420
2421         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
2422
2423         sq->num_to_release += nb_done;
2424
2425         if (likely(!force &&
2426                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
2427                 return;
2428
2429         nb_done = sq->num_to_release;
2430         sq->num_to_release = 0;
2431
2432         for (i = 0; i < nb_done; i++) {
2433                 entry = &sq->ent[sq->tail + num];
2434                 if (unlikely(!entry->buff.addr)) {
2435                         MRVL_LOG(ERR,
2436                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!",
2437                                 sq->tail, (u64)entry->buff.cookie,
2438                                 (u64)entry->buff.addr);
2439                         skip_bufs = 1;
2440                         goto skip;
2441                 }
2442
2443                 if (unlikely(!entry->bpool)) {
2444                         struct rte_mbuf *mbuf;
2445
2446                         mbuf = (struct rte_mbuf *)
2447                                (cookie_addr_high | entry->buff.cookie);
2448                         rte_pktmbuf_free(mbuf);
2449                         skip_bufs = 1;
2450                         goto skip;
2451                 }
2452
2453                 mrvl_port_bpool_size
2454                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
2455                 num++;
2456                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
2457                         goto skip;
2458                 continue;
2459 skip:
2460                 if (likely(num))
2461                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2462                 num += skip_bufs;
2463                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2464                 sq->size -= num;
2465                 num = 0;
2466                 skip_bufs = 0;
2467         }
2468
2469         if (likely(num)) {
2470                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2471                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2472                 sq->size -= num;
2473         }
2474 }
2475
2476 /**
2477  * DPDK callback for transmit.
2478  *
2479  * @param txq
2480  *   Generic pointer transmit queue.
2481  * @param tx_pkts
2482  *   Packets to transmit.
2483  * @param nb_pkts
2484  *   Number of packets in array.
2485  *
2486  * @return
2487  *   Number of packets successfully transmitted.
2488  */
2489 static uint16_t
2490 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2491 {
2492         struct mrvl_txq *q = txq;
2493         struct mrvl_shadow_txq *sq;
2494         struct pp2_hif *hif;
2495         struct pp2_ppio_desc descs[nb_pkts];
2496         unsigned int core_id = rte_lcore_id();
2497         int i, ret, bytes_sent = 0;
2498         uint16_t num, sq_free_size;
2499         uint64_t addr;
2500
2501         hif = mrvl_get_hif(q->priv, core_id);
2502         sq = &q->shadow_txqs[core_id];
2503
2504         if (unlikely(!q->priv->ppio || !hif))
2505                 return 0;
2506
2507         if (sq->size)
2508                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2509                                        sq, q->queue_id, 0);
2510
2511         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2512         if (unlikely(nb_pkts > sq_free_size)) {
2513                 MRVL_LOG(DEBUG,
2514                         "No room in shadow queue for %d packets! %d packets will be sent.",
2515                         nb_pkts, sq_free_size);
2516                 nb_pkts = sq_free_size;
2517         }
2518
2519         for (i = 0; i < nb_pkts; i++) {
2520                 struct rte_mbuf *mbuf = tx_pkts[i];
2521                 int gen_l3_cksum, gen_l4_cksum;
2522                 enum pp2_outq_l3_type l3_type;
2523                 enum pp2_outq_l4_type l4_type;
2524
2525                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2526                         struct rte_mbuf *pref_pkt_hdr;
2527
2528                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2529                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2530                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2531                 }
2532
2533                 mrvl_fill_shadowq(sq, mbuf);
2534                 mrvl_fill_desc(&descs[i], mbuf);
2535
2536                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2537                 /*
2538                  * in case unsupported ol_flags were passed
2539                  * do not update descriptor offload information
2540                  */
2541                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
2542                                               &l3_type, &l4_type, &gen_l3_cksum,
2543                                               &gen_l4_cksum);
2544                 if (unlikely(ret))
2545                         continue;
2546
2547                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2548                                                   mbuf->l2_len,
2549                                                   mbuf->l2_len + mbuf->l3_len,
2550                                                   gen_l3_cksum, gen_l4_cksum);
2551         }
2552
2553         num = nb_pkts;
2554         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2555         /* number of packets that were not sent */
2556         if (unlikely(num > nb_pkts)) {
2557                 for (i = nb_pkts; i < num; i++) {
2558                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2559                                 MRVL_PP2_TX_SHADOWQ_MASK;
2560                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
2561                         bytes_sent -=
2562                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2563                 }
2564                 sq->size -= num - nb_pkts;
2565         }
2566
2567         q->bytes_sent += bytes_sent;
2568
2569         return nb_pkts;
2570 }
2571
2572 /** DPDK callback for S/G transmit.
2573  *
2574  * @param txq
2575  *   Generic pointer transmit queue.
2576  * @param tx_pkts
2577  *   Packets to transmit.
2578  * @param nb_pkts
2579  *   Number of packets in array.
2580  *
2581  * @return
2582  *   Number of packets successfully transmitted.
2583  */
2584 static uint16_t
2585 mrvl_tx_sg_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
2586                      uint16_t nb_pkts)
2587 {
2588         struct mrvl_txq *q = txq;
2589         struct mrvl_shadow_txq *sq;
2590         struct pp2_hif *hif;
2591         struct pp2_ppio_desc descs[nb_pkts * PP2_PPIO_DESC_NUM_FRAGS];
2592         struct pp2_ppio_sg_pkts pkts;
2593         uint8_t frags[nb_pkts];
2594         unsigned int core_id = rte_lcore_id();
2595         int i, j, ret, bytes_sent = 0;
2596         int tail, tail_first;
2597         uint16_t num, sq_free_size;
2598         uint16_t nb_segs, total_descs = 0;
2599         uint64_t addr;
2600
2601         hif = mrvl_get_hif(q->priv, core_id);
2602         sq = &q->shadow_txqs[core_id];
2603         pkts.frags = frags;
2604         pkts.num = 0;
2605
2606         if (unlikely(!q->priv->ppio || !hif))
2607                 return 0;
2608
2609         if (sq->size)
2610                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2611                                        sq, q->queue_id, 0);
2612
2613         /* Save shadow queue free size */
2614         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2615
2616         tail = 0;
2617         for (i = 0; i < nb_pkts; i++) {
2618                 struct rte_mbuf *mbuf = tx_pkts[i];
2619                 struct rte_mbuf *seg = NULL;
2620                 int gen_l3_cksum, gen_l4_cksum;
2621                 enum pp2_outq_l3_type l3_type;
2622                 enum pp2_outq_l4_type l4_type;
2623
2624                 nb_segs = mbuf->nb_segs;
2625                 tail_first = tail;
2626                 total_descs += nb_segs;
2627
2628                 /*
2629                  * Check if total_descs does not exceed
2630                  * shadow queue free size
2631                  */
2632                 if (unlikely(total_descs > sq_free_size)) {
2633                         total_descs -= nb_segs;
2634                         RTE_LOG(DEBUG, PMD,
2635                                 "No room in shadow queue for %d packets! "
2636                                 "%d packets will be sent.\n",
2637                                 nb_pkts, i);
2638                         break;
2639                 }
2640
2641                 /* Check if nb_segs does not exceed the max nb of desc per
2642                  * fragmented packet
2643                  */
2644                 if (nb_segs > PP2_PPIO_DESC_NUM_FRAGS) {
2645                         total_descs -= nb_segs;
2646                         RTE_LOG(ERR, PMD,
2647                                 "Too many segments. Packet won't be sent.\n");
2648                         break;
2649                 }
2650
2651                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2652                         struct rte_mbuf *pref_pkt_hdr;
2653
2654                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2655                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2656                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2657                 }
2658
2659                 pkts.frags[pkts.num] = nb_segs;
2660                 pkts.num++;
2661
2662                 seg = mbuf;
2663                 for (j = 0; j < nb_segs - 1; j++) {
2664                         /* For the subsequent segments, set shadow queue
2665                          * buffer to NULL
2666                          */
2667                         mrvl_fill_shadowq(sq, NULL);
2668                         mrvl_fill_desc(&descs[tail], seg);
2669
2670                         tail++;
2671                         seg = seg->next;
2672                 }
2673                 /* Put first mbuf info in last shadow queue entry */
2674                 mrvl_fill_shadowq(sq, mbuf);
2675                 /* Update descriptor with last segment */
2676                 mrvl_fill_desc(&descs[tail++], seg);
2677
2678                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2679                 /* In case unsupported ol_flags were passed
2680                  * do not update descriptor offload information
2681                  */
2682                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
2683                                               &l3_type, &l4_type, &gen_l3_cksum,
2684                                               &gen_l4_cksum);
2685                 if (unlikely(ret))
2686                         continue;
2687
2688                 pp2_ppio_outq_desc_set_proto_info(&descs[tail_first], l3_type,
2689                                                   l4_type, mbuf->l2_len,
2690                                                   mbuf->l2_len + mbuf->l3_len,
2691                                                   gen_l3_cksum, gen_l4_cksum);
2692         }
2693
2694         num = total_descs;
2695         pp2_ppio_send_sg(q->priv->ppio, hif, q->queue_id, descs,
2696                          &total_descs, &pkts);
2697         /* number of packets that were not sent */
2698         if (unlikely(num > total_descs)) {
2699                 for (i = total_descs; i < num; i++) {
2700                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2701                                 MRVL_PP2_TX_SHADOWQ_MASK;
2702
2703                         addr = sq->ent[sq->head].buff.cookie;
2704                         if (addr)
2705                                 bytes_sent -=
2706                                         rte_pktmbuf_pkt_len((struct rte_mbuf *)
2707                                                 (cookie_addr_high | addr));
2708                 }
2709                 sq->size -= num - total_descs;
2710                 nb_pkts = pkts.num;
2711         }
2712
2713         q->bytes_sent += bytes_sent;
2714
2715         return nb_pkts;
2716 }
2717
2718 /**
2719  * Initialize packet processor.
2720  *
2721  * @return
2722  *   0 on success, negative error value otherwise.
2723  */
2724 static int
2725 mrvl_init_pp2(void)
2726 {
2727         struct pp2_init_params init_params;
2728
2729         memset(&init_params, 0, sizeof(init_params));
2730         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
2731         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
2732         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
2733
2734         return pp2_init(&init_params);
2735 }
2736
2737 /**
2738  * Deinitialize packet processor.
2739  *
2740  * @return
2741  *   0 on success, negative error value otherwise.
2742  */
2743 static void
2744 mrvl_deinit_pp2(void)
2745 {
2746         pp2_deinit();
2747 }
2748
2749 /**
2750  * Create private device structure.
2751  *
2752  * @param dev_name
2753  *   Pointer to the port name passed in the initialization parameters.
2754  *
2755  * @return
2756  *   Pointer to the newly allocated private device structure.
2757  */
2758 static struct mrvl_priv *
2759 mrvl_priv_create(const char *dev_name)
2760 {
2761         struct pp2_bpool_params bpool_params;
2762         char match[MRVL_MATCH_LEN];
2763         struct mrvl_priv *priv;
2764         int ret, bpool_bit;
2765
2766         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
2767         if (!priv)
2768                 return NULL;
2769
2770         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
2771                                        &priv->pp_id, &priv->ppio_id);
2772         if (ret)
2773                 goto out_free_priv;
2774
2775         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
2776                                      PP2_BPOOL_NUM_POOLS);
2777         if (bpool_bit < 0)
2778                 goto out_free_priv;
2779         priv->bpool_bit = bpool_bit;
2780
2781         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
2782                  priv->bpool_bit);
2783         memset(&bpool_params, 0, sizeof(bpool_params));
2784         bpool_params.match = match;
2785         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
2786         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
2787         if (ret)
2788                 goto out_clear_bpool_bit;
2789
2790         priv->ppio_params.type = PP2_PPIO_T_NIC;
2791         rte_spinlock_init(&priv->lock);
2792
2793         return priv;
2794 out_clear_bpool_bit:
2795         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2796 out_free_priv:
2797         rte_free(priv);
2798         return NULL;
2799 }
2800
2801 /**
2802  * Create device representing Ethernet port.
2803  *
2804  * @param name
2805  *   Pointer to the port's name.
2806  *
2807  * @return
2808  *   0 on success, negative error value otherwise.
2809  */
2810 static int
2811 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
2812 {
2813         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
2814         struct rte_eth_dev *eth_dev;
2815         struct mrvl_priv *priv;
2816         struct ifreq req;
2817
2818         eth_dev = rte_eth_dev_allocate(name);
2819         if (!eth_dev)
2820                 return -ENOMEM;
2821
2822         priv = mrvl_priv_create(name);
2823         if (!priv) {
2824                 ret = -ENOMEM;
2825                 goto out_free;
2826         }
2827         eth_dev->data->dev_private = priv;
2828
2829         eth_dev->data->mac_addrs =
2830                 rte_zmalloc("mac_addrs",
2831                             RTE_ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2832         if (!eth_dev->data->mac_addrs) {
2833                 MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
2834                 ret = -ENOMEM;
2835                 goto out_free;
2836         }
2837
2838         memset(&req, 0, sizeof(req));
2839         strcpy(req.ifr_name, name);
2840         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2841         if (ret)
2842                 goto out_free;
2843
2844         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2845                req.ifr_addr.sa_data, RTE_ETHER_ADDR_LEN);
2846
2847         eth_dev->data->kdrv = RTE_KDRV_NONE;
2848         eth_dev->device = &vdev->device;
2849         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2850         mrvl_set_tx_function(eth_dev);
2851         eth_dev->dev_ops = &mrvl_ops;
2852
2853         /* Flag to call rte_eth_dev_release_port() in rte_eth_dev_close(). */
2854         eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
2855
2856         rte_eth_dev_probing_finish(eth_dev);
2857         return 0;
2858 out_free:
2859         rte_eth_dev_release_port(eth_dev);
2860
2861         return ret;
2862 }
2863
2864 /**
2865  * Callback used by rte_kvargs_process() during argument parsing.
2866  *
2867  * @param key
2868  *   Pointer to the parsed key (unused).
2869  * @param value
2870  *   Pointer to the parsed value.
2871  * @param extra_args
2872  *   Pointer to the extra arguments which contains address of the
2873  *   table of pointers to parsed interface names.
2874  *
2875  * @return
2876  *   Always 0.
2877  */
2878 static int
2879 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2880                  void *extra_args)
2881 {
2882         struct mrvl_ifnames *ifnames = extra_args;
2883
2884         ifnames->names[ifnames->idx++] = value;
2885
2886         return 0;
2887 }
2888
2889 /**
2890  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2891  */
2892 static void
2893 mrvl_deinit_hifs(void)
2894 {
2895         int i;
2896
2897         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
2898                 if (hifs[i])
2899                         pp2_hif_deinit(hifs[i]);
2900         }
2901         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
2902         memset(hifs, 0, sizeof(hifs));
2903 }
2904
2905 /**
2906  * DPDK callback to register the virtual device.
2907  *
2908  * @param vdev
2909  *   Pointer to the virtual device.
2910  *
2911  * @return
2912  *   0 on success, negative error value otherwise.
2913  */
2914 static int
2915 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2916 {
2917         struct rte_kvargs *kvlist;
2918         struct mrvl_ifnames ifnames;
2919         int ret = -EINVAL;
2920         uint32_t i, ifnum, cfgnum;
2921         const char *params;
2922
2923         params = rte_vdev_device_args(vdev);
2924         if (!params)
2925                 return -EINVAL;
2926
2927         kvlist = rte_kvargs_parse(params, valid_args);
2928         if (!kvlist)
2929                 return -EINVAL;
2930
2931         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2932         if (ifnum > RTE_DIM(ifnames.names))
2933                 goto out_free_kvlist;
2934
2935         ifnames.idx = 0;
2936         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2937                            mrvl_get_ifnames, &ifnames);
2938
2939
2940         /*
2941          * The below system initialization should be done only once,
2942          * on the first provided configuration file
2943          */
2944         if (!mrvl_qos_cfg) {
2945                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2946                 MRVL_LOG(INFO, "Parsing config file!");
2947                 if (cfgnum > 1) {
2948                         MRVL_LOG(ERR, "Cannot handle more than one config file!");
2949                         goto out_free_kvlist;
2950                 } else if (cfgnum == 1) {
2951                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2952                                            mrvl_get_qoscfg, &mrvl_qos_cfg);
2953                 }
2954         }
2955
2956         if (mrvl_dev_num)
2957                 goto init_devices;
2958
2959         MRVL_LOG(INFO, "Perform MUSDK initializations");
2960
2961         ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist);
2962         if (ret)
2963                 goto out_free_kvlist;
2964
2965         ret = mrvl_init_pp2();
2966         if (ret) {
2967                 MRVL_LOG(ERR, "Failed to init PP!");
2968                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2969                 goto out_free_kvlist;
2970         }
2971
2972         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2973         memset(mrvl_port_to_bpool_lookup, 0, sizeof(mrvl_port_to_bpool_lookup));
2974
2975         mrvl_lcore_first = RTE_MAX_LCORE;
2976         mrvl_lcore_last = 0;
2977
2978 init_devices:
2979         for (i = 0; i < ifnum; i++) {
2980                 MRVL_LOG(INFO, "Creating %s", ifnames.names[i]);
2981                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
2982                 if (ret)
2983                         goto out_cleanup;
2984                 mrvl_dev_num++;
2985         }
2986
2987         rte_kvargs_free(kvlist);
2988
2989         return 0;
2990 out_cleanup:
2991         rte_pmd_mrvl_remove(vdev);
2992
2993 out_free_kvlist:
2994         rte_kvargs_free(kvlist);
2995
2996         return ret;
2997 }
2998
2999 /**
3000  * DPDK callback to remove virtual device.
3001  *
3002  * @param vdev
3003  *   Pointer to the removed virtual device.
3004  *
3005  * @return
3006  *   0 on success, negative error value otherwise.
3007  */
3008 static int
3009 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
3010 {
3011         uint16_t port_id;
3012
3013         RTE_ETH_FOREACH_DEV(port_id) {
3014                 if (rte_eth_devices[port_id].device != &vdev->device)
3015                         continue;
3016                 rte_eth_dev_close(port_id);
3017         }
3018
3019         return 0;
3020 }
3021
3022 static struct rte_vdev_driver pmd_mrvl_drv = {
3023         .probe = rte_pmd_mrvl_probe,
3024         .remove = rte_pmd_mrvl_remove,
3025 };
3026
3027 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv);
3028 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2);
3029
3030 RTE_INIT(mrvl_init_log)
3031 {
3032         mrvl_logtype = rte_log_register("pmd.net.mvpp2");
3033         if (mrvl_logtype >= 0)
3034                 rte_log_set_level(mrvl_logtype, RTE_LOG_NOTICE);
3035 }