ethdev: add return value to stats get dev op
[dpdk.git] / drivers / net / mrvl / mrvl_ethdev.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright(c) 2017 Semihalf. All rights reserved.
5  *
6  *   Redistribution and use in source and binary forms, with or without
7  *   modification, are permitted provided that the following conditions
8  *   are met:
9  *
10  *     * Redistributions of source code must retain the above copyright
11  *       notice, this list of conditions and the following disclaimer.
12  *     * Redistributions in binary form must reproduce the above copyright
13  *       notice, this list of conditions and the following disclaimer in
14  *       the documentation and/or other materials provided with the
15  *       distribution.
16  *     * Neither the name of Semihalf nor the names of its
17  *       contributors may be used to endorse or promote products derived
18  *       from this software without specific prior written permission.
19  *
20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31  */
32
33 #include <rte_ethdev.h>
34 #include <rte_kvargs.h>
35 #include <rte_log.h>
36 #include <rte_malloc.h>
37 #include <rte_vdev.h>
38
39 /* Unluckily, container_of is defined by both DPDK and MUSDK,
40  * we'll declare only one version.
41  *
42  * Note that it is not used in this PMD anyway.
43  */
44 #ifdef container_of
45 #undef container_of
46 #endif
47
48 #include <drivers/mv_pp2.h>
49 #include <drivers/mv_pp2_bpool.h>
50 #include <drivers/mv_pp2_hif.h>
51
52 #include <fcntl.h>
53 #include <linux/ethtool.h>
54 #include <linux/sockios.h>
55 #include <net/if.h>
56 #include <net/if_arp.h>
57 #include <sys/ioctl.h>
58 #include <sys/socket.h>
59 #include <sys/stat.h>
60 #include <sys/types.h>
61
62 #include "mrvl_ethdev.h"
63 #include "mrvl_qos.h"
64
65 /* bitmask with reserved hifs */
66 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
67 /* bitmask with reserved bpools */
68 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
69 /* bitmask with reserved kernel RSS tables */
70 #define MRVL_MUSDK_RSS_RESERVED 0x01
71 /* maximum number of available hifs */
72 #define MRVL_MUSDK_HIFS_MAX 9
73
74 /* prefetch shift */
75 #define MRVL_MUSDK_PREFETCH_SHIFT 2
76
77 /* TCAM has 25 entries reserved for uc/mc filter entries */
78 #define MRVL_MAC_ADDRS_MAX 25
79 #define MRVL_MATCH_LEN 16
80 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
81 /* Maximum allowable packet size */
82 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
83
84 #define MRVL_IFACE_NAME_ARG "iface"
85 #define MRVL_CFG_ARG "cfg"
86
87 #define MRVL_BURST_SIZE 64
88
89 #define MRVL_ARP_LENGTH 28
90
91 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
92
93 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
94 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
95
96 static const char * const valid_args[] = {
97         MRVL_IFACE_NAME_ARG,
98         MRVL_CFG_ARG,
99         NULL
100 };
101
102 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
103 static struct pp2_hif *hifs[RTE_MAX_LCORE];
104 static int used_bpools[PP2_NUM_PKT_PROC] = {
105         MRVL_MUSDK_BPOOLS_RESERVED,
106         MRVL_MUSDK_BPOOLS_RESERVED
107 };
108
109 struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
110 int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
111 uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
112
113 /*
114  * To use buffer harvesting based on loopback port shadow queue structure
115  * was introduced for buffers information bookkeeping.
116  *
117  * Before sending the packet, related buffer information (pp2_buff_inf) is
118  * stored in shadow queue. After packet is transmitted no longer used
119  * packet buffer is released back to it's original hardware pool,
120  * on condition it originated from interface.
121  * In case it  was generated by application itself i.e: mbuf->port field is
122  * 0xff then its released to software mempool.
123  */
124 struct mrvl_shadow_txq {
125         int head;           /* write index - used when sending buffers */
126         int tail;           /* read index - used when releasing buffers */
127         u16 size;           /* queue occupied size */
128         u16 num_to_release; /* number of buffers sent, that can be released */
129         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
130 };
131
132 struct mrvl_rxq {
133         struct mrvl_priv *priv;
134         struct rte_mempool *mp;
135         int queue_id;
136         int port_id;
137         int cksum_enabled;
138         uint64_t bytes_recv;
139         uint64_t drop_mac;
140 };
141
142 struct mrvl_txq {
143         struct mrvl_priv *priv;
144         int queue_id;
145         int port_id;
146         uint64_t bytes_sent;
147 };
148
149 /*
150  * Every tx queue should have dedicated shadow tx queue.
151  *
152  * Ports assigned by DPDK might not start at zero or be continuous so
153  * as a workaround define shadow queues for each possible port so that
154  * we eventually fit somewhere.
155  */
156 struct mrvl_shadow_txq shadow_txqs[RTE_MAX_ETHPORTS][RTE_MAX_LCORE];
157
158 /** Number of ports configured. */
159 int mrvl_ports_nb;
160 static int mrvl_lcore_first;
161 static int mrvl_lcore_last;
162
163 static inline int
164 mrvl_get_bpool_size(int pp2_id, int pool_id)
165 {
166         int i;
167         int size = 0;
168
169         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
170                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
171
172         return size;
173 }
174
175 static inline int
176 mrvl_reserve_bit(int *bitmap, int max)
177 {
178         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
179
180         if (n >= max)
181                 return -1;
182
183         *bitmap |= 1 << n;
184
185         return n;
186 }
187
188 /**
189  * Configure rss based on dpdk rss configuration.
190  *
191  * @param priv
192  *   Pointer to private structure.
193  * @param rss_conf
194  *   Pointer to RSS configuration.
195  *
196  * @return
197  *   0 on success, negative error value otherwise.
198  */
199 static int
200 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
201 {
202         if (rss_conf->rss_key)
203                 RTE_LOG(WARNING, PMD, "Changing hash key is not supported\n");
204
205         if (rss_conf->rss_hf == 0) {
206                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
207         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
208                 priv->ppio_params.inqs_params.hash_type =
209                         PP2_PPIO_HASH_T_2_TUPLE;
210         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
211                 priv->ppio_params.inqs_params.hash_type =
212                         PP2_PPIO_HASH_T_5_TUPLE;
213                 priv->rss_hf_tcp = 1;
214         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
215                 priv->ppio_params.inqs_params.hash_type =
216                         PP2_PPIO_HASH_T_5_TUPLE;
217                 priv->rss_hf_tcp = 0;
218         } else {
219                 return -EINVAL;
220         }
221
222         return 0;
223 }
224
225 /**
226  * Ethernet device configuration.
227  *
228  * Prepare the driver for a given number of TX and RX queues and
229  * configure RSS.
230  *
231  * @param dev
232  *   Pointer to Ethernet device structure.
233  *
234  * @return
235  *   0 on success, negative error value otherwise.
236  */
237 static int
238 mrvl_dev_configure(struct rte_eth_dev *dev)
239 {
240         struct mrvl_priv *priv = dev->data->dev_private;
241         int ret;
242
243         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
244             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
245                 RTE_LOG(INFO, PMD, "Unsupported rx multi queue mode %d\n",
246                         dev->data->dev_conf.rxmode.mq_mode);
247                 return -EINVAL;
248         }
249
250         if (!dev->data->dev_conf.rxmode.hw_strip_crc) {
251                 RTE_LOG(INFO, PMD,
252                         "L2 CRC stripping is always enabled in hw\n");
253                 dev->data->dev_conf.rxmode.hw_strip_crc = 1;
254         }
255
256         if (dev->data->dev_conf.rxmode.hw_vlan_strip) {
257                 RTE_LOG(INFO, PMD, "VLAN stripping not supported\n");
258                 return -EINVAL;
259         }
260
261         if (dev->data->dev_conf.rxmode.split_hdr_size) {
262                 RTE_LOG(INFO, PMD, "Split headers not supported\n");
263                 return -EINVAL;
264         }
265
266         if (dev->data->dev_conf.rxmode.enable_scatter) {
267                 RTE_LOG(INFO, PMD, "RX Scatter/Gather not supported\n");
268                 return -EINVAL;
269         }
270
271         if (dev->data->dev_conf.rxmode.enable_lro) {
272                 RTE_LOG(INFO, PMD, "LRO not supported\n");
273                 return -EINVAL;
274         }
275
276         if (dev->data->dev_conf.rxmode.jumbo_frame)
277                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
278                                  ETHER_HDR_LEN - ETHER_CRC_LEN;
279
280         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
281                                   dev->data->nb_rx_queues);
282         if (ret < 0)
283                 return ret;
284
285         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
286         priv->ppio_params.maintain_stats = 1;
287         priv->nb_rx_queues = dev->data->nb_rx_queues;
288
289         if (dev->data->nb_rx_queues == 1 &&
290             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
291                 RTE_LOG(WARNING, PMD, "Disabling hash for 1 rx queue\n");
292                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
293
294                 return 0;
295         }
296
297         return mrvl_configure_rss(priv,
298                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
299 }
300
301 /**
302  * DPDK callback to change the MTU.
303  *
304  * Setting the MTU affects hardware MRU (packets larger than the MRU
305  * will be dropped).
306  *
307  * @param dev
308  *   Pointer to Ethernet device structure.
309  * @param mtu
310  *   New MTU.
311  *
312  * @return
313  *   0 on success, negative error value otherwise.
314  */
315 static int
316 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
317 {
318         struct mrvl_priv *priv = dev->data->dev_private;
319         /* extra MV_MH_SIZE bytes are required for Marvell tag */
320         uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
321         int ret;
322
323         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
324                 return -EINVAL;
325
326         ret = pp2_ppio_set_mru(priv->ppio, mru);
327         if (ret)
328                 return ret;
329
330         return pp2_ppio_set_mtu(priv->ppio, mtu);
331 }
332
333 /**
334  * DPDK callback to bring the link up.
335  *
336  * @param dev
337  *   Pointer to Ethernet device structure.
338  *
339  * @return
340  *   0 on success, negative error value otherwise.
341  */
342 static int
343 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
344 {
345         struct mrvl_priv *priv = dev->data->dev_private;
346         int ret;
347
348         ret = pp2_ppio_enable(priv->ppio);
349         if (ret)
350                 return ret;
351
352         /*
353          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
354          * as pp2_ppio_enable() changes port->t_mode from default 0 to
355          * PP2_TRAFFIC_INGRESS_EGRESS.
356          *
357          * Set mtu to default DPDK value here.
358          */
359         ret = mrvl_mtu_set(dev, dev->data->mtu);
360         if (ret)
361                 pp2_ppio_disable(priv->ppio);
362
363         dev->data->dev_link.link_status = ETH_LINK_UP;
364
365         return ret;
366 }
367
368 /**
369  * DPDK callback to bring the link down.
370  *
371  * @param dev
372  *   Pointer to Ethernet device structure.
373  *
374  * @return
375  *   0 on success, negative error value otherwise.
376  */
377 static int
378 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
379 {
380         struct mrvl_priv *priv = dev->data->dev_private;
381         int ret;
382
383         ret = pp2_ppio_disable(priv->ppio);
384         if (ret)
385                 return ret;
386
387         dev->data->dev_link.link_status = ETH_LINK_DOWN;
388
389         return ret;
390 }
391
392 /**
393  * DPDK callback to start the device.
394  *
395  * @param dev
396  *   Pointer to Ethernet device structure.
397  *
398  * @return
399  *   0 on success, negative errno value on failure.
400  */
401 static int
402 mrvl_dev_start(struct rte_eth_dev *dev)
403 {
404         struct mrvl_priv *priv = dev->data->dev_private;
405         char match[MRVL_MATCH_LEN];
406         int ret;
407
408         snprintf(match, sizeof(match), "ppio-%d:%d",
409                  priv->pp_id, priv->ppio_id);
410         priv->ppio_params.match = match;
411
412         /*
413          * Calculate the maximum bpool size for refill feature to 1.5 of the
414          * configured size. In case the bpool size will exceed this value,
415          * superfluous buffers will be removed
416          */
417         priv->bpool_max_size = priv->bpool_init_size +
418                               (priv->bpool_init_size >> 1);
419         /*
420          * Calculate the minimum bpool size for refill feature as follows:
421          * 2 default burst sizes multiply by number of rx queues.
422          * If the bpool size will be below this value, new buffers will
423          * be added to the pool.
424          */
425         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
426
427         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
428         if (ret)
429                 return ret;
430
431         /*
432          * In case there are some some stale uc/mc mac addresses flush them
433          * here. It cannot be done during mrvl_dev_close() as port information
434          * is already gone at that point (due to pp2_ppio_deinit() in
435          * mrvl_dev_stop()).
436          */
437         if (!priv->uc_mc_flushed) {
438                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
439                 if (ret) {
440                         RTE_LOG(ERR, PMD,
441                                 "Failed to flush uc/mc filter list\n");
442                         goto out;
443                 }
444                 priv->uc_mc_flushed = 1;
445         }
446
447         if (!priv->vlan_flushed) {
448                 ret = pp2_ppio_flush_vlan(priv->ppio);
449                 if (ret) {
450                         RTE_LOG(ERR, PMD, "Failed to flush vlan list\n");
451                         /*
452                          * TODO
453                          * once pp2_ppio_flush_vlan() is supported jump to out
454                          * goto out;
455                          */
456                 }
457                 priv->vlan_flushed = 1;
458         }
459
460         /* For default QoS config, don't start classifier. */
461         if (mrvl_qos_cfg) {
462                 ret = mrvl_start_qos_mapping(priv);
463                 if (ret) {
464                         pp2_ppio_deinit(priv->ppio);
465                         return ret;
466                 }
467         }
468
469         ret = mrvl_dev_set_link_up(dev);
470         if (ret)
471                 goto out;
472
473         return 0;
474 out:
475         pp2_ppio_deinit(priv->ppio);
476         return ret;
477 }
478
479 /**
480  * Flush receive queues.
481  *
482  * @param dev
483  *   Pointer to Ethernet device structure.
484  */
485 static void
486 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
487 {
488         int i;
489
490         RTE_LOG(INFO, PMD, "Flushing rx queues\n");
491         for (i = 0; i < dev->data->nb_rx_queues; i++) {
492                 int ret, num;
493
494                 do {
495                         struct mrvl_rxq *q = dev->data->rx_queues[i];
496                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
497
498                         num = MRVL_PP2_RXD_MAX;
499                         ret = pp2_ppio_recv(q->priv->ppio,
500                                             q->priv->rxq_map[q->queue_id].tc,
501                                             q->priv->rxq_map[q->queue_id].inq,
502                                             descs, (uint16_t *)&num);
503                 } while (ret == 0 && num);
504         }
505 }
506
507 /**
508  * Flush transmit shadow queues.
509  *
510  * @param dev
511  *   Pointer to Ethernet device structure.
512  */
513 static void
514 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
515 {
516         int i;
517
518         RTE_LOG(INFO, PMD, "Flushing tx shadow queues\n");
519         for (i = 0; i < RTE_MAX_LCORE; i++) {
520                 struct mrvl_shadow_txq *sq =
521                         &shadow_txqs[dev->data->port_id][i];
522
523                 while (sq->tail != sq->head) {
524                         uint64_t addr = cookie_addr_high |
525                                         sq->ent[sq->tail].buff.cookie;
526                         rte_pktmbuf_free((struct rte_mbuf *)addr);
527                         sq->tail = (sq->tail + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
528                 }
529
530                 memset(sq, 0, sizeof(*sq));
531         }
532 }
533
534 /**
535  * Flush hardware bpool (buffer-pool).
536  *
537  * @param dev
538  *   Pointer to Ethernet device structure.
539  */
540 static void
541 mrvl_flush_bpool(struct rte_eth_dev *dev)
542 {
543         struct mrvl_priv *priv = dev->data->dev_private;
544         uint32_t num;
545         int ret;
546
547         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
548         if (ret) {
549                 RTE_LOG(ERR, PMD, "Failed to get bpool buffers number\n");
550                 return;
551         }
552
553         while (num--) {
554                 struct pp2_buff_inf inf;
555                 uint64_t addr;
556
557                 ret = pp2_bpool_get_buff(hifs[rte_lcore_id()], priv->bpool,
558                                          &inf);
559                 if (ret)
560                         break;
561
562                 addr = cookie_addr_high | inf.cookie;
563                 rte_pktmbuf_free((struct rte_mbuf *)addr);
564         }
565 }
566
567 /**
568  * DPDK callback to stop the device.
569  *
570  * @param dev
571  *   Pointer to Ethernet device structure.
572  */
573 static void
574 mrvl_dev_stop(struct rte_eth_dev *dev)
575 {
576         struct mrvl_priv *priv = dev->data->dev_private;
577
578         mrvl_dev_set_link_down(dev);
579         mrvl_flush_rx_queues(dev);
580         mrvl_flush_tx_shadow_queues(dev);
581         if (priv->qos_tbl)
582                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
583         pp2_ppio_deinit(priv->ppio);
584         priv->ppio = NULL;
585 }
586
587 /**
588  * DPDK callback to close the device.
589  *
590  * @param dev
591  *   Pointer to Ethernet device structure.
592  */
593 static void
594 mrvl_dev_close(struct rte_eth_dev *dev)
595 {
596         struct mrvl_priv *priv = dev->data->dev_private;
597         size_t i;
598
599         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
600                 struct pp2_ppio_tc_params *tc_params =
601                         &priv->ppio_params.inqs_params.tcs_params[i];
602
603                 if (tc_params->inqs_params) {
604                         rte_free(tc_params->inqs_params);
605                         tc_params->inqs_params = NULL;
606                 }
607         }
608
609         mrvl_flush_bpool(dev);
610 }
611
612 /**
613  * DPDK callback to retrieve physical link information.
614  *
615  * @param dev
616  *   Pointer to Ethernet device structure.
617  * @param wait_to_complete
618  *   Wait for request completion (ignored).
619  *
620  * @return
621  *   0 on success, negative error value otherwise.
622  */
623 static int
624 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
625 {
626         /*
627          * TODO
628          * once MUSDK provides necessary API use it here
629          */
630         struct ethtool_cmd edata;
631         struct ifreq req;
632         int ret, fd;
633
634         edata.cmd = ETHTOOL_GSET;
635
636         strcpy(req.ifr_name, dev->data->name);
637         req.ifr_data = (void *)&edata;
638
639         fd = socket(AF_INET, SOCK_DGRAM, 0);
640         if (fd == -1)
641                 return -EFAULT;
642
643         ret = ioctl(fd, SIOCETHTOOL, &req);
644         if (ret == -1) {
645                 close(fd);
646                 return -EFAULT;
647         }
648
649         close(fd);
650
651         switch (ethtool_cmd_speed(&edata)) {
652         case SPEED_10:
653                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
654                 break;
655         case SPEED_100:
656                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
657                 break;
658         case SPEED_1000:
659                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
660                 break;
661         case SPEED_10000:
662                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
663                 break;
664         default:
665                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
666         }
667
668         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
669                                                          ETH_LINK_HALF_DUPLEX;
670         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
671                                                            ETH_LINK_FIXED;
672
673         return 0;
674 }
675
676 /**
677  * DPDK callback to enable promiscuous mode.
678  *
679  * @param dev
680  *   Pointer to Ethernet device structure.
681  */
682 static void
683 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
684 {
685         struct mrvl_priv *priv = dev->data->dev_private;
686         int ret;
687
688         ret = pp2_ppio_set_uc_promisc(priv->ppio, 1);
689         if (ret)
690                 RTE_LOG(ERR, PMD, "Failed to enable promiscuous mode\n");
691 }
692
693 /**
694  * DPDK callback to enable allmulti mode.
695  *
696  * @param dev
697  *   Pointer to Ethernet device structure.
698  */
699 static void
700 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
701 {
702         struct mrvl_priv *priv = dev->data->dev_private;
703         int ret;
704
705         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
706         if (ret)
707                 RTE_LOG(ERR, PMD, "Failed enable all-multicast mode\n");
708 }
709
710 /**
711  * DPDK callback to disable promiscuous mode.
712  *
713  * @param dev
714  *   Pointer to Ethernet device structure.
715  */
716 static void
717 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
718 {
719         struct mrvl_priv *priv = dev->data->dev_private;
720         int ret;
721
722         ret = pp2_ppio_set_uc_promisc(priv->ppio, 0);
723         if (ret)
724                 RTE_LOG(ERR, PMD, "Failed to disable promiscuous mode\n");
725 }
726
727 /**
728  * DPDK callback to disable allmulticast mode.
729  *
730  * @param dev
731  *   Pointer to Ethernet device structure.
732  */
733 static void
734 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
735 {
736         struct mrvl_priv *priv = dev->data->dev_private;
737         int ret;
738
739         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
740         if (ret)
741                 RTE_LOG(ERR, PMD, "Failed to disable all-multicast mode\n");
742 }
743
744 /**
745  * DPDK callback to remove a MAC address.
746  *
747  * @param dev
748  *   Pointer to Ethernet device structure.
749  * @param index
750  *   MAC address index.
751  */
752 static void
753 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
754 {
755         struct mrvl_priv *priv = dev->data->dev_private;
756         char buf[ETHER_ADDR_FMT_SIZE];
757         int ret;
758
759         ret = pp2_ppio_remove_mac_addr(priv->ppio,
760                                        dev->data->mac_addrs[index].addr_bytes);
761         if (ret) {
762                 ether_format_addr(buf, sizeof(buf),
763                                   &dev->data->mac_addrs[index]);
764                 RTE_LOG(ERR, PMD, "Failed to remove mac %s\n", buf);
765         }
766 }
767
768 /**
769  * DPDK callback to add a MAC address.
770  *
771  * @param dev
772  *   Pointer to Ethernet device structure.
773  * @param mac_addr
774  *   MAC address to register.
775  * @param index
776  *   MAC address index.
777  * @param vmdq
778  *   VMDq pool index to associate address with (unused).
779  *
780  * @return
781  *   0 on success, negative error value otherwise.
782  */
783 static int
784 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
785                   uint32_t index, uint32_t vmdq __rte_unused)
786 {
787         struct mrvl_priv *priv = dev->data->dev_private;
788         char buf[ETHER_ADDR_FMT_SIZE];
789         int ret;
790
791         if (index == 0)
792                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
793                 return -1;
794
795         /*
796          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
797          * parameter uc_filter_max. Maximum number of mc addresses is then
798          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
799          * 21 respectively.
800          *
801          * If more than uc_filter_max uc addresses were added to filter list
802          * then NIC will switch to promiscuous mode automatically.
803          *
804          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
805          * were added to filter list then NIC will switch to all-multicast mode
806          * automatically.
807          */
808         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
809         if (ret) {
810                 ether_format_addr(buf, sizeof(buf), mac_addr);
811                 RTE_LOG(ERR, PMD, "Failed to add mac %s\n", buf);
812                 return -1;
813         }
814
815         return 0;
816 }
817
818 /**
819  * DPDK callback to set the primary MAC address.
820  *
821  * @param dev
822  *   Pointer to Ethernet device structure.
823  * @param mac_addr
824  *   MAC address to register.
825  */
826 static void
827 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
828 {
829         struct mrvl_priv *priv = dev->data->dev_private;
830
831         pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
832         /*
833          * TODO
834          * Port stops sending packets if pp2_ppio_set_mac_addr()
835          * was called after pp2_ppio_enable(). As a quick fix issue
836          * enable port once again.
837          */
838         pp2_ppio_enable(priv->ppio);
839 }
840
841 /**
842  * DPDK callback to get device statistics.
843  *
844  * @param dev
845  *   Pointer to Ethernet device structure.
846  * @param stats
847  *   Stats structure output buffer.
848  *
849  * @return
850  *   0 on success, negative error value otherwise.
851  */
852 static int
853 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
854 {
855         struct mrvl_priv *priv = dev->data->dev_private;
856         struct pp2_ppio_statistics ppio_stats;
857         uint64_t drop_mac = 0;
858         unsigned int i, idx, ret;
859
860         for (i = 0; i < dev->data->nb_rx_queues; i++) {
861                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
862                 struct pp2_ppio_inq_statistics rx_stats;
863
864                 if (!rxq)
865                         continue;
866
867                 idx = rxq->queue_id;
868                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
869                         RTE_LOG(ERR, PMD,
870                                 "rx queue %d stats out of range (0 - %d)\n",
871                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
872                         continue;
873                 }
874
875                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
876                                                   priv->rxq_map[idx].tc,
877                                                   priv->rxq_map[idx].inq,
878                                                   &rx_stats, 0);
879                 if (unlikely(ret)) {
880                         RTE_LOG(ERR, PMD,
881                                 "Failed to update rx queue %d stats\n", idx);
882                         break;
883                 }
884
885                 stats->q_ibytes[idx] = rxq->bytes_recv;
886                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
887                 stats->q_errors[idx] = rx_stats.drop_early +
888                                        rx_stats.drop_fullq +
889                                        rx_stats.drop_bm +
890                                        rxq->drop_mac;
891                 stats->ibytes += rxq->bytes_recv;
892                 drop_mac += rxq->drop_mac;
893         }
894
895         for (i = 0; i < dev->data->nb_tx_queues; i++) {
896                 struct mrvl_txq *txq = dev->data->tx_queues[i];
897                 struct pp2_ppio_outq_statistics tx_stats;
898
899                 if (!txq)
900                         continue;
901
902                 idx = txq->queue_id;
903                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
904                         RTE_LOG(ERR, PMD,
905                                 "tx queue %d stats out of range (0 - %d)\n",
906                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
907                 }
908
909                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
910                                                    &tx_stats, 0);
911                 if (unlikely(ret)) {
912                         RTE_LOG(ERR, PMD,
913                                 "Failed to update tx queue %d stats\n", idx);
914                         break;
915                 }
916
917                 stats->q_opackets[idx] = tx_stats.deq_desc;
918                 stats->q_obytes[idx] = txq->bytes_sent;
919                 stats->obytes += txq->bytes_sent;
920         }
921
922         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
923         if (unlikely(ret)) {
924                 RTE_LOG(ERR, PMD, "Failed to update port statistics\n");
925                 return ret;
926         }
927
928         stats->ipackets += ppio_stats.rx_packets - drop_mac;
929         stats->opackets += ppio_stats.tx_packets;
930         stats->imissed += ppio_stats.rx_fullq_dropped +
931                           ppio_stats.rx_bm_dropped +
932                           ppio_stats.rx_early_dropped +
933                           ppio_stats.rx_fifo_dropped +
934                           ppio_stats.rx_cls_dropped;
935         stats->ierrors = drop_mac;
936
937         return 0;
938 }
939
940 /**
941  * DPDK callback to clear device statistics.
942  *
943  * @param dev
944  *   Pointer to Ethernet device structure.
945  */
946 static void
947 mrvl_stats_reset(struct rte_eth_dev *dev)
948 {
949         struct mrvl_priv *priv = dev->data->dev_private;
950         int i;
951
952         for (i = 0; i < dev->data->nb_rx_queues; i++) {
953                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
954
955                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
956                                             priv->rxq_map[i].inq, NULL, 1);
957                 rxq->bytes_recv = 0;
958                 rxq->drop_mac = 0;
959         }
960
961         for (i = 0; i < dev->data->nb_tx_queues; i++) {
962                 struct mrvl_txq *txq = dev->data->tx_queues[i];
963
964                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
965                 txq->bytes_sent = 0;
966         }
967
968         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
969 }
970
971 /**
972  * DPDK callback to get information about the device.
973  *
974  * @param dev
975  *   Pointer to Ethernet device structure (unused).
976  * @param info
977  *   Info structure output buffer.
978  */
979 static void
980 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
981                    struct rte_eth_dev_info *info)
982 {
983         info->speed_capa = ETH_LINK_SPEED_10M |
984                            ETH_LINK_SPEED_100M |
985                            ETH_LINK_SPEED_1G |
986                            ETH_LINK_SPEED_10G;
987
988         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
989         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
990         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
991
992         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
993         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
994         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
995
996         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
997         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
998         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
999
1000         info->rx_offload_capa = DEV_RX_OFFLOAD_JUMBO_FRAME |
1001                                 DEV_RX_OFFLOAD_VLAN_FILTER |
1002                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1003                                 DEV_RX_OFFLOAD_UDP_CKSUM |
1004                                 DEV_RX_OFFLOAD_TCP_CKSUM;
1005
1006         info->tx_offload_capa = DEV_TX_OFFLOAD_IPV4_CKSUM |
1007                                 DEV_TX_OFFLOAD_UDP_CKSUM |
1008                                 DEV_TX_OFFLOAD_TCP_CKSUM;
1009
1010         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1011                                        ETH_RSS_NONFRAG_IPV4_TCP |
1012                                        ETH_RSS_NONFRAG_IPV4_UDP;
1013
1014         /* By default packets are dropped if no descriptors are available */
1015         info->default_rxconf.rx_drop_en = 1;
1016
1017         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1018 }
1019
1020 /**
1021  * Return supported packet types.
1022  *
1023  * @param dev
1024  *   Pointer to Ethernet device structure (unused).
1025  *
1026  * @return
1027  *   Const pointer to the table with supported packet types.
1028  */
1029 static const uint32_t *
1030 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1031 {
1032         static const uint32_t ptypes[] = {
1033                 RTE_PTYPE_L2_ETHER,
1034                 RTE_PTYPE_L3_IPV4,
1035                 RTE_PTYPE_L3_IPV4_EXT,
1036                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1037                 RTE_PTYPE_L3_IPV6,
1038                 RTE_PTYPE_L3_IPV6_EXT,
1039                 RTE_PTYPE_L2_ETHER_ARP,
1040                 RTE_PTYPE_L4_TCP,
1041                 RTE_PTYPE_L4_UDP
1042         };
1043
1044         return ptypes;
1045 }
1046
1047 /**
1048  * DPDK callback to get information about specific receive queue.
1049  *
1050  * @param dev
1051  *   Pointer to Ethernet device structure.
1052  * @param rx_queue_id
1053  *   Receive queue index.
1054  * @param qinfo
1055  *   Receive queue information structure.
1056  */
1057 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1058                               struct rte_eth_rxq_info *qinfo)
1059 {
1060         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1061         struct mrvl_priv *priv = dev->data->dev_private;
1062         int inq = priv->rxq_map[rx_queue_id].inq;
1063         int tc = priv->rxq_map[rx_queue_id].tc;
1064         struct pp2_ppio_tc_params *tc_params =
1065                 &priv->ppio_params.inqs_params.tcs_params[tc];
1066
1067         qinfo->mp = q->mp;
1068         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1069 }
1070
1071 /**
1072  * DPDK callback to get information about specific transmit queue.
1073  *
1074  * @param dev
1075  *   Pointer to Ethernet device structure.
1076  * @param tx_queue_id
1077  *   Transmit queue index.
1078  * @param qinfo
1079  *   Transmit queue information structure.
1080  */
1081 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1082                               struct rte_eth_txq_info *qinfo)
1083 {
1084         struct mrvl_priv *priv = dev->data->dev_private;
1085
1086         qinfo->nb_desc =
1087                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1088 }
1089
1090 /**
1091  * DPDK callback to Configure a VLAN filter.
1092  *
1093  * @param dev
1094  *   Pointer to Ethernet device structure.
1095  * @param vlan_id
1096  *   VLAN ID to filter.
1097  * @param on
1098  *   Toggle filter.
1099  *
1100  * @return
1101  *   0 on success, negative error value otherwise.
1102  */
1103 static int
1104 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1105 {
1106         struct mrvl_priv *priv = dev->data->dev_private;
1107
1108         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1109                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1110 }
1111
1112 /**
1113  * Release buffers to hardware bpool (buffer-pool)
1114  *
1115  * @param rxq
1116  *   Receive queue pointer.
1117  * @param num
1118  *   Number of buffers to release to bpool.
1119  *
1120  * @return
1121  *   0 on success, negative error value otherwise.
1122  */
1123 static int
1124 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1125 {
1126         struct buff_release_entry entries[MRVL_PP2_TXD_MAX];
1127         struct rte_mbuf *mbufs[MRVL_PP2_TXD_MAX];
1128         int i, ret;
1129         unsigned int core_id = rte_lcore_id();
1130         struct pp2_hif *hif = hifs[core_id];
1131         struct pp2_bpool *bpool = rxq->priv->bpool;
1132
1133         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1134         if (ret)
1135                 return ret;
1136
1137         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1138                 cookie_addr_high =
1139                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1140
1141         for (i = 0; i < num; i++) {
1142                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1143                         != cookie_addr_high) {
1144                         RTE_LOG(ERR, PMD,
1145                                 "mbuf virtual addr high 0x%lx out of range\n",
1146                                 (uint64_t)mbufs[i] >> 32);
1147                         goto out;
1148                 }
1149
1150                 entries[i].buff.addr =
1151                         rte_mbuf_data_dma_addr_default(mbufs[i]);
1152                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
1153                 entries[i].bpool = bpool;
1154         }
1155
1156         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1157         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1158
1159         if (i != num)
1160                 goto out;
1161
1162         return 0;
1163 out:
1164         for (; i < num; i++)
1165                 rte_pktmbuf_free(mbufs[i]);
1166
1167         return -1;
1168 }
1169
1170 /**
1171  * DPDK callback to configure the receive queue.
1172  *
1173  * @param dev
1174  *   Pointer to Ethernet device structure.
1175  * @param idx
1176  *   RX queue index.
1177  * @param desc
1178  *   Number of descriptors to configure in queue.
1179  * @param socket
1180  *   NUMA socket on which memory must be allocated.
1181  * @param conf
1182  *   Thresholds parameters (unused_).
1183  * @param mp
1184  *   Memory pool for buffer allocations.
1185  *
1186  * @return
1187  *   0 on success, negative error value otherwise.
1188  */
1189 static int
1190 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1191                     unsigned int socket,
1192                     const struct rte_eth_rxconf *conf __rte_unused,
1193                     struct rte_mempool *mp)
1194 {
1195         struct mrvl_priv *priv = dev->data->dev_private;
1196         struct mrvl_rxq *rxq;
1197         uint32_t min_size,
1198                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1199         int ret, tc, inq;
1200
1201         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1202                 /*
1203                  * Unknown TC mapping, mapping will not have a correct queue.
1204                  */
1205                 RTE_LOG(ERR, PMD, "Unknown TC mapping for queue %hu eth%hhu\n",
1206                         idx, priv->ppio_id);
1207                 return -EFAULT;
1208         }
1209
1210         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
1211                    MRVL_PKT_EFFEC_OFFS;
1212         if (min_size < max_rx_pkt_len) {
1213                 RTE_LOG(ERR, PMD,
1214                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.\n",
1215                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
1216                         MRVL_PKT_EFFEC_OFFS,
1217                         max_rx_pkt_len);
1218                 return -EINVAL;
1219         }
1220
1221         if (dev->data->rx_queues[idx]) {
1222                 rte_free(dev->data->rx_queues[idx]);
1223                 dev->data->rx_queues[idx] = NULL;
1224         }
1225
1226         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1227         if (!rxq)
1228                 return -ENOMEM;
1229
1230         rxq->priv = priv;
1231         rxq->mp = mp;
1232         rxq->cksum_enabled = dev->data->dev_conf.rxmode.hw_ip_checksum;
1233         rxq->queue_id = idx;
1234         rxq->port_id = dev->data->port_id;
1235         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1236
1237         tc = priv->rxq_map[rxq->queue_id].tc,
1238         inq = priv->rxq_map[rxq->queue_id].inq;
1239         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1240                 desc;
1241
1242         ret = mrvl_fill_bpool(rxq, desc);
1243         if (ret) {
1244                 rte_free(rxq);
1245                 return ret;
1246         }
1247
1248         priv->bpool_init_size += desc;
1249
1250         dev->data->rx_queues[idx] = rxq;
1251
1252         return 0;
1253 }
1254
1255 /**
1256  * DPDK callback to release the receive queue.
1257  *
1258  * @param rxq
1259  *   Generic receive queue pointer.
1260  */
1261 static void
1262 mrvl_rx_queue_release(void *rxq)
1263 {
1264         struct mrvl_rxq *q = rxq;
1265         struct pp2_ppio_tc_params *tc_params;
1266         int i, num, tc, inq;
1267
1268         if (!q)
1269                 return;
1270
1271         tc = q->priv->rxq_map[q->queue_id].tc;
1272         inq = q->priv->rxq_map[q->queue_id].inq;
1273         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1274         num = tc_params->inqs_params[inq].size;
1275         for (i = 0; i < num; i++) {
1276                 struct pp2_buff_inf inf;
1277                 uint64_t addr;
1278
1279                 pp2_bpool_get_buff(hifs[rte_lcore_id()], q->priv->bpool, &inf);
1280                 addr = cookie_addr_high | inf.cookie;
1281                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1282         }
1283
1284         rte_free(q);
1285 }
1286
1287 /**
1288  * DPDK callback to configure the transmit queue.
1289  *
1290  * @param dev
1291  *   Pointer to Ethernet device structure.
1292  * @param idx
1293  *   Transmit queue index.
1294  * @param desc
1295  *   Number of descriptors to configure in the queue.
1296  * @param socket
1297  *   NUMA socket on which memory must be allocated.
1298  * @param conf
1299  *   Thresholds parameters (unused).
1300  *
1301  * @return
1302  *   0 on success, negative error value otherwise.
1303  */
1304 static int
1305 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1306                     unsigned int socket,
1307                     const struct rte_eth_txconf *conf __rte_unused)
1308 {
1309         struct mrvl_priv *priv = dev->data->dev_private;
1310         struct mrvl_txq *txq;
1311
1312         if (dev->data->tx_queues[idx]) {
1313                 rte_free(dev->data->tx_queues[idx]);
1314                 dev->data->tx_queues[idx] = NULL;
1315         }
1316
1317         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1318         if (!txq)
1319                 return -ENOMEM;
1320
1321         txq->priv = priv;
1322         txq->queue_id = idx;
1323         txq->port_id = dev->data->port_id;
1324         dev->data->tx_queues[idx] = txq;
1325
1326         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1327         priv->ppio_params.outqs_params.outqs_params[idx].weight = 1;
1328
1329         return 0;
1330 }
1331
1332 /**
1333  * DPDK callback to release the transmit queue.
1334  *
1335  * @param txq
1336  *   Generic transmit queue pointer.
1337  */
1338 static void
1339 mrvl_tx_queue_release(void *txq)
1340 {
1341         struct mrvl_txq *q = txq;
1342
1343         if (!q)
1344                 return;
1345
1346         rte_free(q);
1347 }
1348
1349 /**
1350  * Update RSS hash configuration
1351  *
1352  * @param dev
1353  *   Pointer to Ethernet device structure.
1354  * @param rss_conf
1355  *   Pointer to RSS configuration.
1356  *
1357  * @return
1358  *   0 on success, negative error value otherwise.
1359  */
1360 static int
1361 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1362                      struct rte_eth_rss_conf *rss_conf)
1363 {
1364         struct mrvl_priv *priv = dev->data->dev_private;
1365
1366         return mrvl_configure_rss(priv, rss_conf);
1367 }
1368
1369 /**
1370  * DPDK callback to get RSS hash configuration.
1371  *
1372  * @param dev
1373  *   Pointer to Ethernet device structure.
1374  * @rss_conf
1375  *   Pointer to RSS configuration.
1376  *
1377  * @return
1378  *   Always 0.
1379  */
1380 static int
1381 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1382                        struct rte_eth_rss_conf *rss_conf)
1383 {
1384         struct mrvl_priv *priv = dev->data->dev_private;
1385         enum pp2_ppio_hash_type hash_type =
1386                 priv->ppio_params.inqs_params.hash_type;
1387
1388         rss_conf->rss_key = NULL;
1389
1390         if (hash_type == PP2_PPIO_HASH_T_NONE)
1391                 rss_conf->rss_hf = 0;
1392         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1393                 rss_conf->rss_hf = ETH_RSS_IPV4;
1394         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1395                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1396         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1397                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1398
1399         return 0;
1400 }
1401
1402 static const struct eth_dev_ops mrvl_ops = {
1403         .dev_configure = mrvl_dev_configure,
1404         .dev_start = mrvl_dev_start,
1405         .dev_stop = mrvl_dev_stop,
1406         .dev_set_link_up = mrvl_dev_set_link_up,
1407         .dev_set_link_down = mrvl_dev_set_link_down,
1408         .dev_close = mrvl_dev_close,
1409         .link_update = mrvl_link_update,
1410         .promiscuous_enable = mrvl_promiscuous_enable,
1411         .allmulticast_enable = mrvl_allmulticast_enable,
1412         .promiscuous_disable = mrvl_promiscuous_disable,
1413         .allmulticast_disable = mrvl_allmulticast_disable,
1414         .mac_addr_remove = mrvl_mac_addr_remove,
1415         .mac_addr_add = mrvl_mac_addr_add,
1416         .mac_addr_set = mrvl_mac_addr_set,
1417         .mtu_set = mrvl_mtu_set,
1418         .stats_get = mrvl_stats_get,
1419         .stats_reset = mrvl_stats_reset,
1420         .dev_infos_get = mrvl_dev_infos_get,
1421         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
1422         .rxq_info_get = mrvl_rxq_info_get,
1423         .txq_info_get = mrvl_txq_info_get,
1424         .vlan_filter_set = mrvl_vlan_filter_set,
1425         .rx_queue_setup = mrvl_rx_queue_setup,
1426         .rx_queue_release = mrvl_rx_queue_release,
1427         .tx_queue_setup = mrvl_tx_queue_setup,
1428         .tx_queue_release = mrvl_tx_queue_release,
1429         .rss_hash_update = mrvl_rss_hash_update,
1430         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1431 };
1432
1433 /**
1434  * Return packet type information and l3/l4 offsets.
1435  *
1436  * @param desc
1437  *   Pointer to the received packet descriptor.
1438  * @param l3_offset
1439  *   l3 packet offset.
1440  * @param l4_offset
1441  *   l4 packet offset.
1442  *
1443  * @return
1444  *   Packet type information.
1445  */
1446 static inline uint64_t
1447 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
1448                                     uint8_t *l3_offset, uint8_t *l4_offset)
1449 {
1450         enum pp2_inq_l3_type l3_type;
1451         enum pp2_inq_l4_type l4_type;
1452         uint64_t packet_type;
1453
1454         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
1455         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
1456
1457         packet_type = RTE_PTYPE_L2_ETHER;
1458
1459         switch (l3_type) {
1460         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
1461                 packet_type |= RTE_PTYPE_L3_IPV4;
1462                 break;
1463         case PP2_INQ_L3_TYPE_IPV4_OK:
1464                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
1465                 break;
1466         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
1467                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
1468                 break;
1469         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
1470                 packet_type |= RTE_PTYPE_L3_IPV6;
1471                 break;
1472         case PP2_INQ_L3_TYPE_IPV6_EXT:
1473                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
1474                 break;
1475         case PP2_INQ_L3_TYPE_ARP:
1476                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
1477                 /*
1478                  * In case of ARP l4_offset is set to wrong value.
1479                  * Set it to proper one so that later on mbuf->l3_len can be
1480                  * calculated subtracting l4_offset and l3_offset.
1481                  */
1482                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
1483                 break;
1484         default:
1485                 RTE_LOG(DEBUG, PMD, "Failed to recognise l3 packet type\n");
1486                 break;
1487         }
1488
1489         switch (l4_type) {
1490         case PP2_INQ_L4_TYPE_TCP:
1491                 packet_type |= RTE_PTYPE_L4_TCP;
1492                 break;
1493         case PP2_INQ_L4_TYPE_UDP:
1494                 packet_type |= RTE_PTYPE_L4_UDP;
1495                 break;
1496         default:
1497                 RTE_LOG(DEBUG, PMD, "Failed to recognise l4 packet type\n");
1498                 break;
1499         }
1500
1501         return packet_type;
1502 }
1503
1504 /**
1505  * Get offload information from the received packet descriptor.
1506  *
1507  * @param desc
1508  *   Pointer to the received packet descriptor.
1509  *
1510  * @return
1511  *   Mbuf offload flags.
1512  */
1513 static inline uint64_t
1514 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
1515 {
1516         uint64_t flags;
1517         enum pp2_inq_desc_status status;
1518
1519         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
1520         if (unlikely(status != PP2_DESC_ERR_OK))
1521                 flags = PKT_RX_IP_CKSUM_BAD;
1522         else
1523                 flags = PKT_RX_IP_CKSUM_GOOD;
1524
1525         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
1526         if (unlikely(status != PP2_DESC_ERR_OK))
1527                 flags |= PKT_RX_L4_CKSUM_BAD;
1528         else
1529                 flags |= PKT_RX_L4_CKSUM_GOOD;
1530
1531         return flags;
1532 }
1533
1534 /**
1535  * DPDK callback for receive.
1536  *
1537  * @param rxq
1538  *   Generic pointer to the receive queue.
1539  * @param rx_pkts
1540  *   Array to store received packets.
1541  * @param nb_pkts
1542  *   Maximum number of packets in array.
1543  *
1544  * @return
1545  *   Number of packets successfully received.
1546  */
1547 static uint16_t
1548 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1549 {
1550         struct mrvl_rxq *q = rxq;
1551         struct pp2_ppio_desc descs[nb_pkts];
1552         struct pp2_bpool *bpool;
1553         int i, ret, rx_done = 0;
1554         int num;
1555         unsigned int core_id = rte_lcore_id();
1556
1557         if (unlikely(!q->priv->ppio))
1558                 return 0;
1559
1560         bpool = q->priv->bpool;
1561
1562         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
1563                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
1564         if (unlikely(ret < 0)) {
1565                 RTE_LOG(ERR, PMD, "Failed to receive packets\n");
1566                 return 0;
1567         }
1568         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
1569
1570         for (i = 0; i < nb_pkts; i++) {
1571                 struct rte_mbuf *mbuf;
1572                 uint8_t l3_offset, l4_offset;
1573                 enum pp2_inq_desc_status status;
1574                 uint64_t addr;
1575
1576                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1577                         struct pp2_ppio_desc *pref_desc;
1578                         u64 pref_addr;
1579
1580                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
1581                         pref_addr = cookie_addr_high |
1582                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
1583                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
1584                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
1585                 }
1586
1587                 addr = cookie_addr_high |
1588                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
1589                 mbuf = (struct rte_mbuf *)addr;
1590                 rte_pktmbuf_reset(mbuf);
1591
1592                 /* drop packet in case of mac, overrun or resource error */
1593                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
1594                 if (unlikely(status != PP2_DESC_ERR_OK)) {
1595                         struct pp2_buff_inf binf = {
1596                                 .addr = rte_mbuf_data_dma_addr_default(mbuf),
1597                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
1598                         };
1599
1600                         pp2_bpool_put_buff(hifs[core_id], bpool, &binf);
1601                         mrvl_port_bpool_size
1602                                 [bpool->pp2_id][bpool->id][core_id]++;
1603                         q->drop_mac++;
1604                         continue;
1605                 }
1606
1607                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
1608                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
1609                 mbuf->data_len = mbuf->pkt_len;
1610                 mbuf->port = q->port_id;
1611                 mbuf->packet_type =
1612                         mrvl_desc_to_packet_type_and_offset(&descs[i],
1613                                                             &l3_offset,
1614                                                             &l4_offset);
1615                 mbuf->l2_len = l3_offset;
1616                 mbuf->l3_len = l4_offset - l3_offset;
1617
1618                 if (likely(q->cksum_enabled))
1619                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
1620
1621                 rx_pkts[rx_done++] = mbuf;
1622                 q->bytes_recv += mbuf->pkt_len;
1623         }
1624
1625         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
1626                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
1627
1628                 if (unlikely(num <= q->priv->bpool_min_size ||
1629                              (!rx_done && num < q->priv->bpool_init_size))) {
1630                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
1631                         if (ret)
1632                                 RTE_LOG(ERR, PMD, "Failed to fill bpool\n");
1633                 } else if (unlikely(num > q->priv->bpool_max_size)) {
1634                         int i;
1635                         int pkt_to_remove = num - q->priv->bpool_init_size;
1636                         struct rte_mbuf *mbuf;
1637                         struct pp2_buff_inf buff;
1638
1639                         RTE_LOG(DEBUG, PMD,
1640                                 "\nport-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)\n",
1641                                 bpool->pp2_id, q->priv->ppio->port_id,
1642                                 bpool->id, pkt_to_remove, num,
1643                                 q->priv->bpool_init_size);
1644
1645                         for (i = 0; i < pkt_to_remove; i++) {
1646                                 pp2_bpool_get_buff(hifs[core_id], bpool, &buff);
1647                                 mbuf = (struct rte_mbuf *)
1648                                         (cookie_addr_high | buff.cookie);
1649                                 rte_pktmbuf_free(mbuf);
1650                         }
1651                         mrvl_port_bpool_size
1652                                 [bpool->pp2_id][bpool->id][core_id] -=
1653                                                                 pkt_to_remove;
1654                 }
1655                 rte_spinlock_unlock(&q->priv->lock);
1656         }
1657
1658         return rx_done;
1659 }
1660
1661 /**
1662  * Prepare offload information.
1663  *
1664  * @param ol_flags
1665  *   Offload flags.
1666  * @param packet_type
1667  *   Packet type bitfield.
1668  * @param l3_type
1669  *   Pointer to the pp2_ouq_l3_type structure.
1670  * @param l4_type
1671  *   Pointer to the pp2_outq_l4_type structure.
1672  * @param gen_l3_cksum
1673  *   Will be set to 1 in case l3 checksum is computed.
1674  * @param l4_cksum
1675  *   Will be set to 1 in case l4 checksum is computed.
1676  *
1677  * @return
1678  *   0 on success, negative error value otherwise.
1679  */
1680 static inline int
1681 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
1682                         enum pp2_outq_l3_type *l3_type,
1683                         enum pp2_outq_l4_type *l4_type,
1684                         int *gen_l3_cksum,
1685                         int *gen_l4_cksum)
1686 {
1687         /*
1688          * Based on ol_flags prepare information
1689          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
1690          * for offloading.
1691          */
1692         if (ol_flags & PKT_TX_IPV4) {
1693                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
1694                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
1695         } else if (ol_flags & PKT_TX_IPV6) {
1696                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
1697                 /* no checksum for ipv6 header */
1698                 *gen_l3_cksum = 0;
1699         } else {
1700                 /* if something different then stop processing */
1701                 return -1;
1702         }
1703
1704         ol_flags &= PKT_TX_L4_MASK;
1705         if ((packet_type & RTE_PTYPE_L4_TCP) &&
1706             ol_flags == PKT_TX_TCP_CKSUM) {
1707                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
1708                 *gen_l4_cksum = 1;
1709         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
1710                    ol_flags == PKT_TX_UDP_CKSUM) {
1711                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
1712                 *gen_l4_cksum = 1;
1713         } else {
1714                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
1715                 /* no checksum for other type */
1716                 *gen_l4_cksum = 0;
1717         }
1718
1719         return 0;
1720 }
1721
1722 /**
1723  * Release already sent buffers to bpool (buffer-pool).
1724  *
1725  * @param ppio
1726  *   Pointer to the port structure.
1727  * @param hif
1728  *   Pointer to the MUSDK hardware interface.
1729  * @param sq
1730  *   Pointer to the shadow queue.
1731  * @param qid
1732  *   Queue id number.
1733  * @param force
1734  *   Force releasing packets.
1735  */
1736 static inline void
1737 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
1738                        struct mrvl_shadow_txq *sq, int qid, int force)
1739 {
1740         struct buff_release_entry *entry;
1741         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
1742         int i, core_id = rte_lcore_id();
1743
1744         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
1745
1746         sq->num_to_release += nb_done;
1747
1748         if (likely(!force &&
1749                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
1750                 return;
1751
1752         nb_done = sq->num_to_release;
1753         sq->num_to_release = 0;
1754
1755         for (i = 0; i < nb_done; i++) {
1756                 entry = &sq->ent[sq->tail + num];
1757                 if (unlikely(!entry->buff.addr)) {
1758                         RTE_LOG(ERR, PMD,
1759                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!\n",
1760                                 sq->tail, (u64)entry->buff.cookie,
1761                                 (u64)entry->buff.addr);
1762                         skip_bufs = 1;
1763                         goto skip;
1764                 }
1765
1766                 if (unlikely(!entry->bpool)) {
1767                         struct rte_mbuf *mbuf;
1768
1769                         mbuf = (struct rte_mbuf *)
1770                                (cookie_addr_high | entry->buff.cookie);
1771                         rte_pktmbuf_free(mbuf);
1772                         skip_bufs = 1;
1773                         goto skip;
1774                 }
1775
1776                 mrvl_port_bpool_size
1777                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
1778                 num++;
1779                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
1780                         goto skip;
1781                 continue;
1782 skip:
1783                 if (likely(num))
1784                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1785                 num += skip_bufs;
1786                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1787                 sq->size -= num;
1788                 num = 0;
1789         }
1790
1791         if (likely(num)) {
1792                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1793                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1794                 sq->size -= num;
1795         }
1796 }
1797
1798 /**
1799  * DPDK callback for transmit.
1800  *
1801  * @param txq
1802  *   Generic pointer transmit queue.
1803  * @param tx_pkts
1804  *   Packets to transmit.
1805  * @param nb_pkts
1806  *   Number of packets in array.
1807  *
1808  * @return
1809  *   Number of packets successfully transmitted.
1810  */
1811 static uint16_t
1812 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1813 {
1814         struct mrvl_txq *q = txq;
1815         struct mrvl_shadow_txq *sq = &shadow_txqs[q->port_id][rte_lcore_id()];
1816         struct pp2_hif *hif = hifs[rte_lcore_id()];
1817         struct pp2_ppio_desc descs[nb_pkts];
1818         int i, ret, bytes_sent = 0;
1819         uint16_t num, sq_free_size;
1820         uint64_t addr;
1821
1822         if (unlikely(!q->priv->ppio))
1823                 return 0;
1824
1825         if (sq->size)
1826                 mrvl_free_sent_buffers(q->priv->ppio, hif, sq, q->queue_id, 0);
1827
1828         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
1829         if (unlikely(nb_pkts > sq_free_size)) {
1830                 RTE_LOG(DEBUG, PMD,
1831                         "No room in shadow queue for %d packets! %d packets will be sent.\n",
1832                         nb_pkts, sq_free_size);
1833                 nb_pkts = sq_free_size;
1834         }
1835
1836         for (i = 0; i < nb_pkts; i++) {
1837                 struct rte_mbuf *mbuf = tx_pkts[i];
1838                 int gen_l3_cksum, gen_l4_cksum;
1839                 enum pp2_outq_l3_type l3_type;
1840                 enum pp2_outq_l4_type l4_type;
1841
1842                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1843                         struct rte_mbuf *pref_pkt_hdr;
1844
1845                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
1846                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
1847                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
1848                 }
1849
1850                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
1851                 sq->ent[sq->head].buff.addr =
1852                         rte_mbuf_data_dma_addr_default(mbuf);
1853                 sq->ent[sq->head].bpool =
1854                         (unlikely(mbuf->port == 0xff || mbuf->refcnt > 1)) ?
1855                          NULL : mrvl_port_to_bpool_lookup[mbuf->port];
1856                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
1857                 sq->size++;
1858
1859                 pp2_ppio_outq_desc_reset(&descs[i]);
1860                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
1861                                                  rte_pktmbuf_mtophys(mbuf));
1862                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
1863                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
1864                                                rte_pktmbuf_pkt_len(mbuf));
1865
1866                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
1867                 /*
1868                  * in case unsupported ol_flags were passed
1869                  * do not update descriptor offload information
1870                  */
1871                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
1872                                               &l3_type, &l4_type, &gen_l3_cksum,
1873                                               &gen_l4_cksum);
1874                 if (unlikely(ret))
1875                         continue;
1876
1877                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
1878                                                   mbuf->l2_len,
1879                                                   mbuf->l2_len + mbuf->l3_len,
1880                                                   gen_l3_cksum, gen_l4_cksum);
1881         }
1882
1883         num = nb_pkts;
1884         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
1885         /* number of packets that were not sent */
1886         if (unlikely(num > nb_pkts)) {
1887                 for (i = nb_pkts; i < num; i++) {
1888                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
1889                                 MRVL_PP2_TX_SHADOWQ_MASK;
1890                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
1891                         bytes_sent -=
1892                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
1893                 }
1894                 sq->size -= num - nb_pkts;
1895         }
1896
1897         q->bytes_sent += bytes_sent;
1898
1899         return nb_pkts;
1900 }
1901
1902 /**
1903  * Initialize packet processor.
1904  *
1905  * @return
1906  *   0 on success, negative error value otherwise.
1907  */
1908 static int
1909 mrvl_init_pp2(void)
1910 {
1911         struct pp2_init_params init_params;
1912
1913         memset(&init_params, 0, sizeof(init_params));
1914         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
1915         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
1916         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
1917
1918         return pp2_init(&init_params);
1919 }
1920
1921 /**
1922  * Deinitialize packet processor.
1923  *
1924  * @return
1925  *   0 on success, negative error value otherwise.
1926  */
1927 static void
1928 mrvl_deinit_pp2(void)
1929 {
1930         pp2_deinit();
1931 }
1932
1933 /**
1934  * Create private device structure.
1935  *
1936  * @param dev_name
1937  *   Pointer to the port name passed in the initialization parameters.
1938  *
1939  * @return
1940  *   Pointer to the newly allocated private device structure.
1941  */
1942 static struct mrvl_priv *
1943 mrvl_priv_create(const char *dev_name)
1944 {
1945         struct pp2_bpool_params bpool_params;
1946         char match[MRVL_MATCH_LEN];
1947         struct mrvl_priv *priv;
1948         int ret, bpool_bit;
1949
1950         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
1951         if (!priv)
1952                 return NULL;
1953
1954         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
1955                                        &priv->pp_id, &priv->ppio_id);
1956         if (ret)
1957                 goto out_free_priv;
1958
1959         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
1960                                      PP2_BPOOL_NUM_POOLS);
1961         if (bpool_bit < 0)
1962                 goto out_free_priv;
1963         priv->bpool_bit = bpool_bit;
1964
1965         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
1966                  priv->bpool_bit);
1967         memset(&bpool_params, 0, sizeof(bpool_params));
1968         bpool_params.match = match;
1969         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
1970         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
1971         if (ret)
1972                 goto out_clear_bpool_bit;
1973
1974         priv->ppio_params.type = PP2_PPIO_T_NIC;
1975         rte_spinlock_init(&priv->lock);
1976
1977         return priv;
1978 out_clear_bpool_bit:
1979         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
1980 out_free_priv:
1981         rte_free(priv);
1982         return NULL;
1983 }
1984
1985 /**
1986  * Create device representing Ethernet port.
1987  *
1988  * @param name
1989  *   Pointer to the port's name.
1990  *
1991  * @return
1992  *   0 on success, negative error value otherwise.
1993  */
1994 static int
1995 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
1996 {
1997         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
1998         struct rte_eth_dev *eth_dev;
1999         struct mrvl_priv *priv;
2000         struct ifreq req;
2001
2002         eth_dev = rte_eth_dev_allocate(name);
2003         if (!eth_dev)
2004                 return -ENOMEM;
2005
2006         priv = mrvl_priv_create(name);
2007         if (!priv) {
2008                 ret = -ENOMEM;
2009                 goto out_free_dev;
2010         }
2011
2012         eth_dev->data->mac_addrs =
2013                 rte_zmalloc("mac_addrs",
2014                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2015         if (!eth_dev->data->mac_addrs) {
2016                 RTE_LOG(ERR, PMD, "Failed to allocate space for eth addrs\n");
2017                 ret = -ENOMEM;
2018                 goto out_free_priv;
2019         }
2020
2021         memset(&req, 0, sizeof(req));
2022         strcpy(req.ifr_name, name);
2023         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2024         if (ret)
2025                 goto out_free_mac;
2026
2027         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2028                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2029
2030         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2031         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
2032         eth_dev->data->dev_private = priv;
2033         eth_dev->device = &vdev->device;
2034         eth_dev->dev_ops = &mrvl_ops;
2035
2036         return 0;
2037 out_free_mac:
2038         rte_free(eth_dev->data->mac_addrs);
2039 out_free_dev:
2040         rte_eth_dev_release_port(eth_dev);
2041 out_free_priv:
2042         rte_free(priv);
2043
2044         return ret;
2045 }
2046
2047 /**
2048  * Cleanup previously created device representing Ethernet port.
2049  *
2050  * @param name
2051  *   Pointer to the port name.
2052  */
2053 static void
2054 mrvl_eth_dev_destroy(const char *name)
2055 {
2056         struct rte_eth_dev *eth_dev;
2057         struct mrvl_priv *priv;
2058
2059         eth_dev = rte_eth_dev_allocated(name);
2060         if (!eth_dev)
2061                 return;
2062
2063         priv = eth_dev->data->dev_private;
2064         pp2_bpool_deinit(priv->bpool);
2065         rte_free(priv);
2066         rte_free(eth_dev->data->mac_addrs);
2067         rte_eth_dev_release_port(eth_dev);
2068 }
2069
2070 /**
2071  * Callback used by rte_kvargs_process() during argument parsing.
2072  *
2073  * @param key
2074  *   Pointer to the parsed key (unused).
2075  * @param value
2076  *   Pointer to the parsed value.
2077  * @param extra_args
2078  *   Pointer to the extra arguments which contains address of the
2079  *   table of pointers to parsed interface names.
2080  *
2081  * @return
2082  *   Always 0.
2083  */
2084 static int
2085 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2086                  void *extra_args)
2087 {
2088         const char **ifnames = extra_args;
2089
2090         ifnames[mrvl_ports_nb++] = value;
2091
2092         return 0;
2093 }
2094
2095 /**
2096  * Initialize per-lcore MUSDK hardware interfaces (hifs).
2097  *
2098  * @return
2099  *   0 on success, negative error value otherwise.
2100  */
2101 static int
2102 mrvl_init_hifs(void)
2103 {
2104         struct pp2_hif_params params;
2105         char match[MRVL_MATCH_LEN];
2106         int i, ret;
2107
2108         RTE_LCORE_FOREACH(i) {
2109                 ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
2110                 if (ret < 0)
2111                         return ret;
2112
2113                 snprintf(match, sizeof(match), "hif-%d", ret);
2114                 memset(&params, 0, sizeof(params));
2115                 params.match = match;
2116                 params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
2117                 ret = pp2_hif_init(&params, &hifs[i]);
2118                 if (ret) {
2119                         RTE_LOG(ERR, PMD, "Failed to initialize hif %d\n", i);
2120                         return ret;
2121                 }
2122         }
2123
2124         return 0;
2125 }
2126
2127 /**
2128  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2129  */
2130 static void
2131 mrvl_deinit_hifs(void)
2132 {
2133         int i;
2134
2135         RTE_LCORE_FOREACH(i) {
2136                 if (hifs[i])
2137                         pp2_hif_deinit(hifs[i]);
2138         }
2139 }
2140
2141 static void mrvl_set_first_last_cores(int core_id)
2142 {
2143         if (core_id < mrvl_lcore_first)
2144                 mrvl_lcore_first = core_id;
2145
2146         if (core_id > mrvl_lcore_last)
2147                 mrvl_lcore_last = core_id;
2148 }
2149
2150 /**
2151  * DPDK callback to register the virtual device.
2152  *
2153  * @param vdev
2154  *   Pointer to the virtual device.
2155  *
2156  * @return
2157  *   0 on success, negative error value otherwise.
2158  */
2159 static int
2160 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2161 {
2162         struct rte_kvargs *kvlist;
2163         const char *ifnames[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
2164         int ret = -EINVAL;
2165         uint32_t i, ifnum, cfgnum, core_id;
2166         const char *params;
2167
2168         params = rte_vdev_device_args(vdev);
2169         if (!params)
2170                 return -EINVAL;
2171
2172         kvlist = rte_kvargs_parse(params, valid_args);
2173         if (!kvlist)
2174                 return -EINVAL;
2175
2176         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2177         if (ifnum > RTE_DIM(ifnames))
2178                 goto out_free_kvlist;
2179
2180         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2181                            mrvl_get_ifnames, &ifnames);
2182
2183         cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2184         if (cfgnum > 1) {
2185                 RTE_LOG(ERR, PMD, "Cannot handle more than one config file!\n");
2186                 goto out_free_kvlist;
2187         } else if (cfgnum == 1) {
2188                 rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2189                                    mrvl_get_qoscfg, &mrvl_qos_cfg);
2190         }
2191
2192         /*
2193          * ret == -EEXIST is correct, it means DMA
2194          * has been already initialized (by another PMD).
2195          */
2196         ret = mv_sys_dma_mem_init(RTE_MRVL_MUSDK_DMA_MEMSIZE);
2197         if (ret < 0 && ret != -EEXIST)
2198                 goto out_free_kvlist;
2199
2200         ret = mrvl_init_pp2();
2201         if (ret) {
2202                 RTE_LOG(ERR, PMD, "Failed to init PP!\n");
2203                 goto out_deinit_dma;
2204         }
2205
2206         ret = mrvl_init_hifs();
2207         if (ret)
2208                 goto out_deinit_hifs;
2209
2210         for (i = 0; i < ifnum; i++) {
2211                 RTE_LOG(INFO, PMD, "Creating %s\n", ifnames[i]);
2212                 ret = mrvl_eth_dev_create(vdev, ifnames[i]);
2213                 if (ret)
2214                         goto out_cleanup;
2215         }
2216
2217         rte_kvargs_free(kvlist);
2218
2219         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2220
2221         mrvl_lcore_first = RTE_MAX_LCORE;
2222         mrvl_lcore_last = 0;
2223
2224         RTE_LCORE_FOREACH(core_id) {
2225                 mrvl_set_first_last_cores(core_id);
2226         }
2227
2228         return 0;
2229 out_cleanup:
2230         for (; i > 0; i--)
2231                 mrvl_eth_dev_destroy(ifnames[i]);
2232 out_deinit_hifs:
2233         mrvl_deinit_hifs();
2234         mrvl_deinit_pp2();
2235 out_deinit_dma:
2236         mv_sys_dma_mem_destroy();
2237 out_free_kvlist:
2238         rte_kvargs_free(kvlist);
2239
2240         return ret;
2241 }
2242
2243 /**
2244  * DPDK callback to remove virtual device.
2245  *
2246  * @param vdev
2247  *   Pointer to the removed virtual device.
2248  *
2249  * @return
2250  *   0 on success, negative error value otherwise.
2251  */
2252 static int
2253 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2254 {
2255         int i;
2256         const char *name;
2257
2258         name = rte_vdev_device_name(vdev);
2259         if (!name)
2260                 return -EINVAL;
2261
2262         RTE_LOG(INFO, PMD, "Removing %s\n", name);
2263
2264         for (i = 0; i < rte_eth_dev_count(); i++) {
2265                 char ifname[RTE_ETH_NAME_MAX_LEN];
2266
2267                 rte_eth_dev_get_name_by_port(i, ifname);
2268                 mrvl_eth_dev_destroy(ifname);
2269         }
2270
2271         mrvl_deinit_hifs();
2272         mrvl_deinit_pp2();
2273         mv_sys_dma_mem_destroy();
2274
2275         return 0;
2276 }
2277
2278 static struct rte_vdev_driver pmd_mrvl_drv = {
2279         .probe = rte_pmd_mrvl_probe,
2280         .remove = rte_pmd_mrvl_remove,
2281 };
2282
2283 RTE_PMD_REGISTER_VDEV(net_mrvl, pmd_mrvl_drv);
2284 RTE_PMD_REGISTER_ALIAS(net_mrvl, eth_mrvl);