03bf458ad931097e3d71c5ffc130374bd5efad43
[dpdk.git] / drivers / net / thunderx / nicvf_ethdev.c
1 /*
2  *   BSD LICENSE
3  *
4  *   Copyright (C) Cavium networks Ltd. 2016.
5  *
6  *   Redistribution and use in source and binary forms, with or without
7  *   modification, are permitted provided that the following conditions
8  *   are met:
9  *
10  *     * Redistributions of source code must retain the above copyright
11  *       notice, this list of conditions and the following disclaimer.
12  *     * Redistributions in binary form must reproduce the above copyright
13  *       notice, this list of conditions and the following disclaimer in
14  *       the documentation and/or other materials provided with the
15  *       distribution.
16  *     * Neither the name of Cavium networks nor the names of its
17  *       contributors may be used to endorse or promote products derived
18  *       from this software without specific prior written permission.
19  *
20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31  */
32
33 #include <assert.h>
34 #include <stdio.h>
35 #include <stdbool.h>
36 #include <errno.h>
37 #include <stdint.h>
38 #include <string.h>
39 #include <unistd.h>
40 #include <stdarg.h>
41 #include <inttypes.h>
42 #include <netinet/in.h>
43 #include <sys/queue.h>
44 #include <sys/timerfd.h>
45
46 #include <rte_alarm.h>
47 #include <rte_atomic.h>
48 #include <rte_branch_prediction.h>
49 #include <rte_byteorder.h>
50 #include <rte_common.h>
51 #include <rte_cycles.h>
52 #include <rte_debug.h>
53 #include <rte_dev.h>
54 #include <rte_eal.h>
55 #include <rte_ether.h>
56 #include <rte_ethdev.h>
57 #include <rte_interrupts.h>
58 #include <rte_log.h>
59 #include <rte_memory.h>
60 #include <rte_memzone.h>
61 #include <rte_malloc.h>
62 #include <rte_random.h>
63 #include <rte_pci.h>
64 #include <rte_tailq.h>
65
66 #include "base/nicvf_plat.h"
67
68 #include "nicvf_ethdev.h"
69 #include "nicvf_rxtx.h"
70 #include "nicvf_logs.h"
71
72 static void nicvf_dev_stop(struct rte_eth_dev *dev);
73
74 static inline int
75 nicvf_atomic_write_link_status(struct rte_eth_dev *dev,
76                                struct rte_eth_link *link)
77 {
78         struct rte_eth_link *dst = &dev->data->dev_link;
79         struct rte_eth_link *src = link;
80
81         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
82                 *(uint64_t *)src) == 0)
83                 return -1;
84
85         return 0;
86 }
87
88 static inline void
89 nicvf_set_eth_link_status(struct nicvf *nic, struct rte_eth_link *link)
90 {
91         link->link_status = nic->link_up;
92         link->link_duplex = ETH_LINK_AUTONEG;
93         if (nic->duplex == NICVF_HALF_DUPLEX)
94                 link->link_duplex = ETH_LINK_HALF_DUPLEX;
95         else if (nic->duplex == NICVF_FULL_DUPLEX)
96                 link->link_duplex = ETH_LINK_FULL_DUPLEX;
97         link->link_speed = nic->speed;
98         link->link_autoneg = ETH_LINK_SPEED_AUTONEG;
99 }
100
101 static void
102 nicvf_interrupt(void *arg)
103 {
104         struct nicvf *nic = arg;
105
106         if (nicvf_reg_poll_interrupts(nic) == NIC_MBOX_MSG_BGX_LINK_CHANGE) {
107                 if (nic->eth_dev->data->dev_conf.intr_conf.lsc)
108                         nicvf_set_eth_link_status(nic,
109                                         &nic->eth_dev->data->dev_link);
110                 _rte_eth_dev_callback_process(nic->eth_dev,
111                                 RTE_ETH_EVENT_INTR_LSC);
112         }
113
114         rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
115                                 nicvf_interrupt, nic);
116 }
117
118 static int
119 nicvf_periodic_alarm_start(struct nicvf *nic)
120 {
121         return rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
122                                         nicvf_interrupt, nic);
123 }
124
125 static int
126 nicvf_periodic_alarm_stop(struct nicvf *nic)
127 {
128         return rte_eal_alarm_cancel(nicvf_interrupt, nic);
129 }
130
131 /*
132  * Return 0 means link status changed, -1 means not changed
133  */
134 static int
135 nicvf_dev_link_update(struct rte_eth_dev *dev,
136                       int wait_to_complete __rte_unused)
137 {
138         struct rte_eth_link link;
139         struct nicvf *nic = nicvf_pmd_priv(dev);
140
141         PMD_INIT_FUNC_TRACE();
142
143         memset(&link, 0, sizeof(link));
144         nicvf_set_eth_link_status(nic, &link);
145         return nicvf_atomic_write_link_status(dev, &link);
146 }
147
148 static int
149 nicvf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
150 {
151         struct nicvf *nic = nicvf_pmd_priv(dev);
152         uint32_t buffsz, frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
153
154         PMD_INIT_FUNC_TRACE();
155
156         if (frame_size > NIC_HW_MAX_FRS)
157                 return -EINVAL;
158
159         if (frame_size < NIC_HW_MIN_FRS)
160                 return -EINVAL;
161
162         buffsz = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
163
164         /*
165          * Refuse mtu that requires the support of scattered packets
166          * when this feature has not been enabled before.
167          */
168         if (!dev->data->scattered_rx &&
169                 (frame_size + 2 * VLAN_TAG_SIZE > buffsz))
170                 return -EINVAL;
171
172         /* check <seg size> * <max_seg>  >= max_frame */
173         if (dev->data->scattered_rx &&
174                 (frame_size + 2 * VLAN_TAG_SIZE > buffsz * NIC_HW_MAX_SEGS))
175                 return -EINVAL;
176
177         if (frame_size > ETHER_MAX_LEN)
178                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
179         else
180                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
181
182         if (nicvf_mbox_update_hw_max_frs(nic, frame_size))
183                 return -EINVAL;
184
185         /* Update max frame size */
186         dev->data->dev_conf.rxmode.max_rx_pkt_len = (uint32_t)frame_size;
187         nic->mtu = mtu;
188         return 0;
189 }
190
191 static int
192 nicvf_dev_get_reg_length(struct rte_eth_dev *dev  __rte_unused)
193 {
194         return nicvf_reg_get_count();
195 }
196
197 static int
198 nicvf_dev_get_regs(struct rte_eth_dev *dev, struct rte_dev_reg_info *regs)
199 {
200         uint64_t *data = regs->data;
201         struct nicvf *nic = nicvf_pmd_priv(dev);
202
203         if (data == NULL)
204                 return -EINVAL;
205
206         /* Support only full register dump */
207         if ((regs->length == 0) ||
208                 (regs->length == (uint32_t)nicvf_reg_get_count())) {
209                 regs->version = nic->vendor_id << 16 | nic->device_id;
210                 nicvf_reg_dump(nic, data);
211                 return 0;
212         }
213         return -ENOTSUP;
214 }
215
216 static void
217 nicvf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
218 {
219         uint16_t qidx;
220         struct nicvf_hw_rx_qstats rx_qstats;
221         struct nicvf_hw_tx_qstats tx_qstats;
222         struct nicvf_hw_stats port_stats;
223         struct nicvf *nic = nicvf_pmd_priv(dev);
224
225         /* Reading per RX ring stats */
226         for (qidx = 0; qidx < dev->data->nb_rx_queues; qidx++) {
227                 if (qidx == RTE_ETHDEV_QUEUE_STAT_CNTRS)
228                         break;
229
230                 nicvf_hw_get_rx_qstats(nic, &rx_qstats, qidx);
231                 stats->q_ibytes[qidx] = rx_qstats.q_rx_bytes;
232                 stats->q_ipackets[qidx] = rx_qstats.q_rx_packets;
233         }
234
235         /* Reading per TX ring stats */
236         for (qidx = 0; qidx < dev->data->nb_tx_queues; qidx++) {
237                 if (qidx == RTE_ETHDEV_QUEUE_STAT_CNTRS)
238                         break;
239
240                 nicvf_hw_get_tx_qstats(nic, &tx_qstats, qidx);
241                 stats->q_obytes[qidx] = tx_qstats.q_tx_bytes;
242                 stats->q_opackets[qidx] = tx_qstats.q_tx_packets;
243         }
244
245         nicvf_hw_get_stats(nic, &port_stats);
246         stats->ibytes = port_stats.rx_bytes;
247         stats->ipackets = port_stats.rx_ucast_frames;
248         stats->ipackets += port_stats.rx_bcast_frames;
249         stats->ipackets += port_stats.rx_mcast_frames;
250         stats->ierrors = port_stats.rx_l2_errors;
251         stats->imissed = port_stats.rx_drop_red;
252         stats->imissed += port_stats.rx_drop_overrun;
253         stats->imissed += port_stats.rx_drop_bcast;
254         stats->imissed += port_stats.rx_drop_mcast;
255         stats->imissed += port_stats.rx_drop_l3_bcast;
256         stats->imissed += port_stats.rx_drop_l3_mcast;
257
258         stats->obytes = port_stats.tx_bytes_ok;
259         stats->opackets = port_stats.tx_ucast_frames_ok;
260         stats->opackets += port_stats.tx_bcast_frames_ok;
261         stats->opackets += port_stats.tx_mcast_frames_ok;
262         stats->oerrors = port_stats.tx_drops;
263 }
264
265 static const uint32_t *
266 nicvf_dev_supported_ptypes_get(struct rte_eth_dev *dev)
267 {
268         size_t copied;
269         static uint32_t ptypes[32];
270         struct nicvf *nic = nicvf_pmd_priv(dev);
271         static const uint32_t ptypes_pass1[] = {
272                 RTE_PTYPE_L3_IPV4,
273                 RTE_PTYPE_L3_IPV4_EXT,
274                 RTE_PTYPE_L3_IPV6,
275                 RTE_PTYPE_L3_IPV6_EXT,
276                 RTE_PTYPE_L4_TCP,
277                 RTE_PTYPE_L4_UDP,
278                 RTE_PTYPE_L4_FRAG,
279         };
280         static const uint32_t ptypes_pass2[] = {
281                 RTE_PTYPE_TUNNEL_GRE,
282                 RTE_PTYPE_TUNNEL_GENEVE,
283                 RTE_PTYPE_TUNNEL_VXLAN,
284                 RTE_PTYPE_TUNNEL_NVGRE,
285         };
286         static const uint32_t ptypes_end = RTE_PTYPE_UNKNOWN;
287
288         copied = sizeof(ptypes_pass1);
289         memcpy(ptypes, ptypes_pass1, copied);
290         if (nicvf_hw_version(nic) == NICVF_PASS2) {
291                 memcpy((char *)ptypes + copied, ptypes_pass2,
292                         sizeof(ptypes_pass2));
293                 copied += sizeof(ptypes_pass2);
294         }
295
296         memcpy((char *)ptypes + copied, &ptypes_end, sizeof(ptypes_end));
297         if (dev->rx_pkt_burst == nicvf_recv_pkts ||
298                 dev->rx_pkt_burst == nicvf_recv_pkts_multiseg)
299                 return ptypes;
300
301         return NULL;
302 }
303
304 static void
305 nicvf_dev_stats_reset(struct rte_eth_dev *dev)
306 {
307         int i;
308         uint16_t rxqs = 0, txqs = 0;
309         struct nicvf *nic = nicvf_pmd_priv(dev);
310
311         for (i = 0; i < dev->data->nb_rx_queues; i++)
312                 rxqs |= (0x3 << (i * 2));
313         for (i = 0; i < dev->data->nb_tx_queues; i++)
314                 txqs |= (0x3 << (i * 2));
315
316         nicvf_mbox_reset_stat_counters(nic, 0x3FFF, 0x1F, rxqs, txqs);
317 }
318
319 /* Promiscuous mode enabled by default in LMAC to VF 1:1 map configuration */
320 static void
321 nicvf_dev_promisc_enable(struct rte_eth_dev *dev __rte_unused)
322 {
323 }
324
325 static inline uint64_t
326 nicvf_rss_ethdev_to_nic(struct nicvf *nic, uint64_t ethdev_rss)
327 {
328         uint64_t nic_rss = 0;
329
330         if (ethdev_rss & ETH_RSS_IPV4)
331                 nic_rss |= RSS_IP_ENA;
332
333         if (ethdev_rss & ETH_RSS_IPV6)
334                 nic_rss |= RSS_IP_ENA;
335
336         if (ethdev_rss & ETH_RSS_NONFRAG_IPV4_UDP)
337                 nic_rss |= (RSS_IP_ENA | RSS_UDP_ENA);
338
339         if (ethdev_rss & ETH_RSS_NONFRAG_IPV4_TCP)
340                 nic_rss |= (RSS_IP_ENA | RSS_TCP_ENA);
341
342         if (ethdev_rss & ETH_RSS_NONFRAG_IPV6_UDP)
343                 nic_rss |= (RSS_IP_ENA | RSS_UDP_ENA);
344
345         if (ethdev_rss & ETH_RSS_NONFRAG_IPV6_TCP)
346                 nic_rss |= (RSS_IP_ENA | RSS_TCP_ENA);
347
348         if (ethdev_rss & ETH_RSS_PORT)
349                 nic_rss |= RSS_L2_EXTENDED_HASH_ENA;
350
351         if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING) {
352                 if (ethdev_rss & ETH_RSS_VXLAN)
353                         nic_rss |= RSS_TUN_VXLAN_ENA;
354
355                 if (ethdev_rss & ETH_RSS_GENEVE)
356                         nic_rss |= RSS_TUN_GENEVE_ENA;
357
358                 if (ethdev_rss & ETH_RSS_NVGRE)
359                         nic_rss |= RSS_TUN_NVGRE_ENA;
360         }
361
362         return nic_rss;
363 }
364
365 static inline uint64_t
366 nicvf_rss_nic_to_ethdev(struct nicvf *nic,  uint64_t nic_rss)
367 {
368         uint64_t ethdev_rss = 0;
369
370         if (nic_rss & RSS_IP_ENA)
371                 ethdev_rss |= (ETH_RSS_IPV4 | ETH_RSS_IPV6);
372
373         if ((nic_rss & RSS_IP_ENA) && (nic_rss & RSS_TCP_ENA))
374                 ethdev_rss |= (ETH_RSS_NONFRAG_IPV4_TCP |
375                                 ETH_RSS_NONFRAG_IPV6_TCP);
376
377         if ((nic_rss & RSS_IP_ENA) && (nic_rss & RSS_UDP_ENA))
378                 ethdev_rss |= (ETH_RSS_NONFRAG_IPV4_UDP |
379                                 ETH_RSS_NONFRAG_IPV6_UDP);
380
381         if (nic_rss & RSS_L2_EXTENDED_HASH_ENA)
382                 ethdev_rss |= ETH_RSS_PORT;
383
384         if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING) {
385                 if (nic_rss & RSS_TUN_VXLAN_ENA)
386                         ethdev_rss |= ETH_RSS_VXLAN;
387
388                 if (nic_rss & RSS_TUN_GENEVE_ENA)
389                         ethdev_rss |= ETH_RSS_GENEVE;
390
391                 if (nic_rss & RSS_TUN_NVGRE_ENA)
392                         ethdev_rss |= ETH_RSS_NVGRE;
393         }
394         return ethdev_rss;
395 }
396
397 static int
398 nicvf_dev_reta_query(struct rte_eth_dev *dev,
399                      struct rte_eth_rss_reta_entry64 *reta_conf,
400                      uint16_t reta_size)
401 {
402         struct nicvf *nic = nicvf_pmd_priv(dev);
403         uint8_t tbl[NIC_MAX_RSS_IDR_TBL_SIZE];
404         int ret, i, j;
405
406         if (reta_size != NIC_MAX_RSS_IDR_TBL_SIZE) {
407                 RTE_LOG(ERR, PMD, "The size of hash lookup table configured "
408                         "(%d) doesn't match the number hardware can supported "
409                         "(%d)", reta_size, NIC_MAX_RSS_IDR_TBL_SIZE);
410                 return -EINVAL;
411         }
412
413         ret = nicvf_rss_reta_query(nic, tbl, NIC_MAX_RSS_IDR_TBL_SIZE);
414         if (ret)
415                 return ret;
416
417         /* Copy RETA table */
418         for (i = 0; i < (NIC_MAX_RSS_IDR_TBL_SIZE / RTE_RETA_GROUP_SIZE); i++) {
419                 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
420                         if ((reta_conf[i].mask >> j) & 0x01)
421                                 reta_conf[i].reta[j] = tbl[j];
422         }
423
424         return 0;
425 }
426
427 static int
428 nicvf_dev_reta_update(struct rte_eth_dev *dev,
429                       struct rte_eth_rss_reta_entry64 *reta_conf,
430                       uint16_t reta_size)
431 {
432         struct nicvf *nic = nicvf_pmd_priv(dev);
433         uint8_t tbl[NIC_MAX_RSS_IDR_TBL_SIZE];
434         int ret, i, j;
435
436         if (reta_size != NIC_MAX_RSS_IDR_TBL_SIZE) {
437                 RTE_LOG(ERR, PMD, "The size of hash lookup table configured "
438                         "(%d) doesn't match the number hardware can supported "
439                         "(%d)", reta_size, NIC_MAX_RSS_IDR_TBL_SIZE);
440                 return -EINVAL;
441         }
442
443         ret = nicvf_rss_reta_query(nic, tbl, NIC_MAX_RSS_IDR_TBL_SIZE);
444         if (ret)
445                 return ret;
446
447         /* Copy RETA table */
448         for (i = 0; i < (NIC_MAX_RSS_IDR_TBL_SIZE / RTE_RETA_GROUP_SIZE); i++) {
449                 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
450                         if ((reta_conf[i].mask >> j) & 0x01)
451                                 tbl[j] = reta_conf[i].reta[j];
452         }
453
454         return nicvf_rss_reta_update(nic, tbl, NIC_MAX_RSS_IDR_TBL_SIZE);
455 }
456
457 static int
458 nicvf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
459                             struct rte_eth_rss_conf *rss_conf)
460 {
461         struct nicvf *nic = nicvf_pmd_priv(dev);
462
463         if (rss_conf->rss_key)
464                 nicvf_rss_get_key(nic, rss_conf->rss_key);
465
466         rss_conf->rss_key_len =  RSS_HASH_KEY_BYTE_SIZE;
467         rss_conf->rss_hf = nicvf_rss_nic_to_ethdev(nic, nicvf_rss_get_cfg(nic));
468         return 0;
469 }
470
471 static int
472 nicvf_dev_rss_hash_update(struct rte_eth_dev *dev,
473                           struct rte_eth_rss_conf *rss_conf)
474 {
475         struct nicvf *nic = nicvf_pmd_priv(dev);
476         uint64_t nic_rss;
477
478         if (rss_conf->rss_key &&
479                 rss_conf->rss_key_len != RSS_HASH_KEY_BYTE_SIZE) {
480                 RTE_LOG(ERR, PMD, "Hash key size mismatch %d",
481                                 rss_conf->rss_key_len);
482                 return -EINVAL;
483         }
484
485         if (rss_conf->rss_key)
486                 nicvf_rss_set_key(nic, rss_conf->rss_key);
487
488         nic_rss = nicvf_rss_ethdev_to_nic(nic, rss_conf->rss_hf);
489         nicvf_rss_set_cfg(nic, nic_rss);
490         return 0;
491 }
492
493 static int
494 nicvf_qset_cq_alloc(struct nicvf *nic, struct nicvf_rxq *rxq, uint16_t qidx,
495                     uint32_t desc_cnt)
496 {
497         const struct rte_memzone *rz;
498         uint32_t ring_size = desc_cnt * sizeof(union cq_entry_t);
499
500         rz = rte_eth_dma_zone_reserve(nic->eth_dev, "cq_ring", qidx, ring_size,
501                                         NICVF_CQ_BASE_ALIGN_BYTES, nic->node);
502         if (rz == NULL) {
503                 PMD_INIT_LOG(ERR, "Failed to allocate mem for cq hw ring");
504                 return -ENOMEM;
505         }
506
507         memset(rz->addr, 0, ring_size);
508
509         rxq->phys = rz->phys_addr;
510         rxq->desc = rz->addr;
511         rxq->qlen_mask = desc_cnt - 1;
512
513         return 0;
514 }
515
516 static int
517 nicvf_qset_sq_alloc(struct nicvf *nic,  struct nicvf_txq *sq, uint16_t qidx,
518                     uint32_t desc_cnt)
519 {
520         const struct rte_memzone *rz;
521         uint32_t ring_size = desc_cnt * sizeof(union sq_entry_t);
522
523         rz = rte_eth_dma_zone_reserve(nic->eth_dev, "sq", qidx, ring_size,
524                                 NICVF_SQ_BASE_ALIGN_BYTES, nic->node);
525         if (rz == NULL) {
526                 PMD_INIT_LOG(ERR, "Failed allocate mem for sq hw ring");
527                 return -ENOMEM;
528         }
529
530         memset(rz->addr, 0, ring_size);
531
532         sq->phys = rz->phys_addr;
533         sq->desc = rz->addr;
534         sq->qlen_mask = desc_cnt - 1;
535
536         return 0;
537 }
538
539 static int
540 nicvf_qset_rbdr_alloc(struct nicvf *nic, uint32_t desc_cnt, uint32_t buffsz)
541 {
542         struct nicvf_rbdr *rbdr;
543         const struct rte_memzone *rz;
544         uint32_t ring_size;
545
546         assert(nic->rbdr == NULL);
547         rbdr = rte_zmalloc_socket("rbdr", sizeof(struct nicvf_rbdr),
548                                   RTE_CACHE_LINE_SIZE, nic->node);
549         if (rbdr == NULL) {
550                 PMD_INIT_LOG(ERR, "Failed to allocate mem for rbdr");
551                 return -ENOMEM;
552         }
553
554         ring_size = sizeof(struct rbdr_entry_t) * desc_cnt;
555         rz = rte_eth_dma_zone_reserve(nic->eth_dev, "rbdr", 0, ring_size,
556                                    NICVF_RBDR_BASE_ALIGN_BYTES, nic->node);
557         if (rz == NULL) {
558                 PMD_INIT_LOG(ERR, "Failed to allocate mem for rbdr desc ring");
559                 return -ENOMEM;
560         }
561
562         memset(rz->addr, 0, ring_size);
563
564         rbdr->phys = rz->phys_addr;
565         rbdr->tail = 0;
566         rbdr->next_tail = 0;
567         rbdr->desc = rz->addr;
568         rbdr->buffsz = buffsz;
569         rbdr->qlen_mask = desc_cnt - 1;
570         rbdr->rbdr_status =
571                 nicvf_qset_base(nic, 0) + NIC_QSET_RBDR_0_1_STATUS0;
572         rbdr->rbdr_door =
573                 nicvf_qset_base(nic, 0) + NIC_QSET_RBDR_0_1_DOOR;
574
575         nic->rbdr = rbdr;
576         return 0;
577 }
578
579 static void
580 nicvf_rbdr_release_mbuf(struct nicvf *nic, nicvf_phys_addr_t phy)
581 {
582         uint16_t qidx;
583         void *obj;
584         struct nicvf_rxq *rxq;
585
586         for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) {
587                 rxq = nic->eth_dev->data->rx_queues[qidx];
588                 if (rxq->precharge_cnt) {
589                         obj = (void *)nicvf_mbuff_phy2virt(phy,
590                                                            rxq->mbuf_phys_off);
591                         rte_mempool_put(rxq->pool, obj);
592                         rxq->precharge_cnt--;
593                         break;
594                 }
595         }
596 }
597
598 static inline void
599 nicvf_rbdr_release_mbufs(struct nicvf *nic)
600 {
601         uint32_t qlen_mask, head;
602         struct rbdr_entry_t *entry;
603         struct nicvf_rbdr *rbdr = nic->rbdr;
604
605         qlen_mask = rbdr->qlen_mask;
606         head = rbdr->head;
607         while (head != rbdr->tail) {
608                 entry = rbdr->desc + head;
609                 nicvf_rbdr_release_mbuf(nic, entry->full_addr);
610                 head++;
611                 head = head & qlen_mask;
612         }
613 }
614
615 static inline void
616 nicvf_tx_queue_release_mbufs(struct nicvf_txq *txq)
617 {
618         uint32_t head;
619
620         head = txq->head;
621         while (head != txq->tail) {
622                 if (txq->txbuffs[head]) {
623                         rte_pktmbuf_free_seg(txq->txbuffs[head]);
624                         txq->txbuffs[head] = NULL;
625                 }
626                 head++;
627                 head = head & txq->qlen_mask;
628         }
629 }
630
631 static void
632 nicvf_tx_queue_reset(struct nicvf_txq *txq)
633 {
634         uint32_t txq_desc_cnt = txq->qlen_mask + 1;
635
636         memset(txq->desc, 0, sizeof(union sq_entry_t) * txq_desc_cnt);
637         memset(txq->txbuffs, 0, sizeof(struct rte_mbuf *) * txq_desc_cnt);
638         txq->tail = 0;
639         txq->head = 0;
640         txq->xmit_bufs = 0;
641 }
642
643 static inline int
644 nicvf_start_tx_queue(struct rte_eth_dev *dev, uint16_t qidx)
645 {
646         struct nicvf_txq *txq;
647         int ret;
648
649         if (dev->data->tx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STARTED)
650                 return 0;
651
652         txq = dev->data->tx_queues[qidx];
653         txq->pool = NULL;
654         ret = nicvf_qset_sq_config(nicvf_pmd_priv(dev), qidx, txq);
655         if (ret) {
656                 PMD_INIT_LOG(ERR, "Failed to configure sq %d %d", qidx, ret);
657                 goto config_sq_error;
658         }
659
660         dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STARTED;
661         return ret;
662
663 config_sq_error:
664         nicvf_qset_sq_reclaim(nicvf_pmd_priv(dev), qidx);
665         return ret;
666 }
667
668 static inline int
669 nicvf_stop_tx_queue(struct rte_eth_dev *dev, uint16_t qidx)
670 {
671         struct nicvf_txq *txq;
672         int ret;
673
674         if (dev->data->tx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STOPPED)
675                 return 0;
676
677         ret = nicvf_qset_sq_reclaim(nicvf_pmd_priv(dev), qidx);
678         if (ret)
679                 PMD_INIT_LOG(ERR, "Failed to reclaim sq %d %d", qidx, ret);
680
681         txq = dev->data->tx_queues[qidx];
682         nicvf_tx_queue_release_mbufs(txq);
683         nicvf_tx_queue_reset(txq);
684
685         dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED;
686         return ret;
687 }
688
689 static inline int
690 nicvf_configure_cpi(struct rte_eth_dev *dev)
691 {
692         struct nicvf *nic = nicvf_pmd_priv(dev);
693         uint16_t qidx, qcnt;
694         int ret;
695
696         /* Count started rx queues */
697         for (qidx = qcnt = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++)
698                 if (dev->data->rx_queue_state[qidx] ==
699                     RTE_ETH_QUEUE_STATE_STARTED)
700                         qcnt++;
701
702         nic->cpi_alg = CPI_ALG_NONE;
703         ret = nicvf_mbox_config_cpi(nic, qcnt);
704         if (ret)
705                 PMD_INIT_LOG(ERR, "Failed to configure CPI %d", ret);
706
707         return ret;
708 }
709
710 static inline int
711 nicvf_configure_rss(struct rte_eth_dev *dev)
712 {
713         struct nicvf *nic = nicvf_pmd_priv(dev);
714         uint64_t rsshf;
715         int ret = -EINVAL;
716
717         rsshf = nicvf_rss_ethdev_to_nic(nic,
718                         dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf);
719         PMD_DRV_LOG(INFO, "mode=%d rx_queues=%d loopback=%d rsshf=0x%" PRIx64,
720                     dev->data->dev_conf.rxmode.mq_mode,
721                     nic->eth_dev->data->nb_rx_queues,
722                     nic->eth_dev->data->dev_conf.lpbk_mode, rsshf);
723
724         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_NONE)
725                 ret = nicvf_rss_term(nic);
726         else if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS)
727                 ret = nicvf_rss_config(nic,
728                                        nic->eth_dev->data->nb_rx_queues, rsshf);
729         if (ret)
730                 PMD_INIT_LOG(ERR, "Failed to configure RSS %d", ret);
731
732         return ret;
733 }
734
735 static int
736 nicvf_configure_rss_reta(struct rte_eth_dev *dev)
737 {
738         struct nicvf *nic = nicvf_pmd_priv(dev);
739         unsigned int idx, qmap_size;
740         uint8_t qmap[RTE_MAX_QUEUES_PER_PORT];
741         uint8_t default_reta[NIC_MAX_RSS_IDR_TBL_SIZE];
742
743         if (nic->cpi_alg != CPI_ALG_NONE)
744                 return -EINVAL;
745
746         /* Prepare queue map */
747         for (idx = 0, qmap_size = 0; idx < dev->data->nb_rx_queues; idx++) {
748                 if (dev->data->rx_queue_state[idx] ==
749                                 RTE_ETH_QUEUE_STATE_STARTED)
750                         qmap[qmap_size++] = idx;
751         }
752
753         /* Update default RSS RETA */
754         for (idx = 0; idx < NIC_MAX_RSS_IDR_TBL_SIZE; idx++)
755                 default_reta[idx] = qmap[idx % qmap_size];
756
757         return nicvf_rss_reta_update(nic, default_reta,
758                                      NIC_MAX_RSS_IDR_TBL_SIZE);
759 }
760
761 static void
762 nicvf_dev_tx_queue_release(void *sq)
763 {
764         struct nicvf_txq *txq;
765
766         PMD_INIT_FUNC_TRACE();
767
768         txq = (struct nicvf_txq *)sq;
769         if (txq) {
770                 if (txq->txbuffs != NULL) {
771                         nicvf_tx_queue_release_mbufs(txq);
772                         rte_free(txq->txbuffs);
773                         txq->txbuffs = NULL;
774                 }
775                 rte_free(txq);
776         }
777 }
778
779 static void
780 nicvf_set_tx_function(struct rte_eth_dev *dev)
781 {
782         struct nicvf_txq *txq;
783         size_t i;
784         bool multiseg = false;
785
786         for (i = 0; i < dev->data->nb_tx_queues; i++) {
787                 txq = dev->data->tx_queues[i];
788                 if ((txq->txq_flags & ETH_TXQ_FLAGS_NOMULTSEGS) == 0) {
789                         multiseg = true;
790                         break;
791                 }
792         }
793
794         /* Use a simple Tx queue (no offloads, no multi segs) if possible */
795         if (multiseg) {
796                 PMD_DRV_LOG(DEBUG, "Using multi-segment tx callback");
797                 dev->tx_pkt_burst = nicvf_xmit_pkts_multiseg;
798         } else {
799                 PMD_DRV_LOG(DEBUG, "Using single-segment tx callback");
800                 dev->tx_pkt_burst = nicvf_xmit_pkts;
801         }
802
803         if (txq->pool_free == nicvf_single_pool_free_xmited_buffers)
804                 PMD_DRV_LOG(DEBUG, "Using single-mempool tx free method");
805         else
806                 PMD_DRV_LOG(DEBUG, "Using multi-mempool tx free method");
807 }
808
809 static void
810 nicvf_set_rx_function(struct rte_eth_dev *dev)
811 {
812         if (dev->data->scattered_rx) {
813                 PMD_DRV_LOG(DEBUG, "Using multi-segment rx callback");
814                 dev->rx_pkt_burst = nicvf_recv_pkts_multiseg;
815         } else {
816                 PMD_DRV_LOG(DEBUG, "Using single-segment rx callback");
817                 dev->rx_pkt_burst = nicvf_recv_pkts;
818         }
819 }
820
821 static int
822 nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
823                          uint16_t nb_desc, unsigned int socket_id,
824                          const struct rte_eth_txconf *tx_conf)
825 {
826         uint16_t tx_free_thresh;
827         uint8_t is_single_pool;
828         struct nicvf_txq *txq;
829         struct nicvf *nic = nicvf_pmd_priv(dev);
830
831         PMD_INIT_FUNC_TRACE();
832
833         /* Socket id check */
834         if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node)
835                 PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d",
836                 socket_id, nic->node);
837
838         /* Tx deferred start is not supported */
839         if (tx_conf->tx_deferred_start) {
840                 PMD_INIT_LOG(ERR, "Tx deferred start not supported");
841                 return -EINVAL;
842         }
843
844         /* Roundup nb_desc to available qsize and validate max number of desc */
845         nb_desc = nicvf_qsize_sq_roundup(nb_desc);
846         if (nb_desc == 0) {
847                 PMD_INIT_LOG(ERR, "Value of nb_desc beyond available sq qsize");
848                 return -EINVAL;
849         }
850
851         /* Validate tx_free_thresh */
852         tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
853                                 tx_conf->tx_free_thresh :
854                                 NICVF_DEFAULT_TX_FREE_THRESH);
855
856         if (tx_free_thresh > (nb_desc) ||
857                 tx_free_thresh > NICVF_MAX_TX_FREE_THRESH) {
858                 PMD_INIT_LOG(ERR,
859                         "tx_free_thresh must be less than the number of TX "
860                         "descriptors. (tx_free_thresh=%u port=%d "
861                         "queue=%d)", (unsigned int)tx_free_thresh,
862                         (int)dev->data->port_id, (int)qidx);
863                 return -EINVAL;
864         }
865
866         /* Free memory prior to re-allocation if needed. */
867         if (dev->data->tx_queues[qidx] != NULL) {
868                 PMD_TX_LOG(DEBUG, "Freeing memory prior to re-allocation %d",
869                                 qidx);
870                 nicvf_dev_tx_queue_release(dev->data->tx_queues[qidx]);
871                 dev->data->tx_queues[qidx] = NULL;
872         }
873
874         /* Allocating tx queue data structure */
875         txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct nicvf_txq),
876                                         RTE_CACHE_LINE_SIZE, nic->node);
877         if (txq == NULL) {
878                 PMD_INIT_LOG(ERR, "Failed to allocate txq=%d", qidx);
879                 return -ENOMEM;
880         }
881
882         txq->nic = nic;
883         txq->queue_id = qidx;
884         txq->tx_free_thresh = tx_free_thresh;
885         txq->txq_flags = tx_conf->txq_flags;
886         txq->sq_head = nicvf_qset_base(nic, qidx) + NIC_QSET_SQ_0_7_HEAD;
887         txq->sq_door = nicvf_qset_base(nic, qidx) + NIC_QSET_SQ_0_7_DOOR;
888         is_single_pool = (txq->txq_flags & ETH_TXQ_FLAGS_NOREFCOUNT &&
889                                 txq->txq_flags & ETH_TXQ_FLAGS_NOMULTMEMP);
890
891         /* Choose optimum free threshold value for multipool case */
892         if (!is_single_pool) {
893                 txq->tx_free_thresh = (uint16_t)
894                 (tx_conf->tx_free_thresh == NICVF_DEFAULT_TX_FREE_THRESH ?
895                                 NICVF_TX_FREE_MPOOL_THRESH :
896                                 tx_conf->tx_free_thresh);
897                 txq->pool_free = nicvf_multi_pool_free_xmited_buffers;
898         } else {
899                 txq->pool_free = nicvf_single_pool_free_xmited_buffers;
900         }
901
902         /* Allocate software ring */
903         txq->txbuffs = rte_zmalloc_socket("txq->txbuffs",
904                                 nb_desc * sizeof(struct rte_mbuf *),
905                                 RTE_CACHE_LINE_SIZE, nic->node);
906
907         if (txq->txbuffs == NULL) {
908                 nicvf_dev_tx_queue_release(txq);
909                 return -ENOMEM;
910         }
911
912         if (nicvf_qset_sq_alloc(nic, txq, qidx, nb_desc)) {
913                 PMD_INIT_LOG(ERR, "Failed to allocate mem for sq %d", qidx);
914                 nicvf_dev_tx_queue_release(txq);
915                 return -ENOMEM;
916         }
917
918         nicvf_tx_queue_reset(txq);
919
920         PMD_TX_LOG(DEBUG, "[%d] txq=%p nb_desc=%d desc=%p phys=0x%" PRIx64,
921                         qidx, txq, nb_desc, txq->desc, txq->phys);
922
923         dev->data->tx_queues[qidx] = txq;
924         dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED;
925         return 0;
926 }
927
928 static inline void
929 nicvf_rx_queue_release_mbufs(struct nicvf_rxq *rxq)
930 {
931         uint32_t rxq_cnt;
932         uint32_t nb_pkts, released_pkts = 0;
933         uint32_t refill_cnt = 0;
934         struct rte_eth_dev *dev = rxq->nic->eth_dev;
935         struct rte_mbuf *rx_pkts[NICVF_MAX_RX_FREE_THRESH];
936
937         if (dev->rx_pkt_burst == NULL)
938                 return;
939
940         while ((rxq_cnt = nicvf_dev_rx_queue_count(dev, rxq->queue_id))) {
941                 nb_pkts = dev->rx_pkt_burst(rxq, rx_pkts,
942                                         NICVF_MAX_RX_FREE_THRESH);
943                 PMD_DRV_LOG(INFO, "nb_pkts=%d  rxq_cnt=%d", nb_pkts, rxq_cnt);
944                 while (nb_pkts) {
945                         rte_pktmbuf_free_seg(rx_pkts[--nb_pkts]);
946                         released_pkts++;
947                 }
948         }
949
950         refill_cnt += nicvf_dev_rbdr_refill(dev, rxq->queue_id);
951         PMD_DRV_LOG(INFO, "free_cnt=%d  refill_cnt=%d",
952                     released_pkts, refill_cnt);
953 }
954
955 static void
956 nicvf_rx_queue_reset(struct nicvf_rxq *rxq)
957 {
958         rxq->head = 0;
959         rxq->available_space = 0;
960         rxq->recv_buffers = 0;
961 }
962
963 static inline int
964 nicvf_start_rx_queue(struct rte_eth_dev *dev, uint16_t qidx)
965 {
966         struct nicvf *nic = nicvf_pmd_priv(dev);
967         struct nicvf_rxq *rxq;
968         int ret;
969
970         if (dev->data->rx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STARTED)
971                 return 0;
972
973         /* Update rbdr pointer to all rxq */
974         rxq = dev->data->rx_queues[qidx];
975         rxq->shared_rbdr = nic->rbdr;
976
977         ret = nicvf_qset_rq_config(nic, qidx, rxq);
978         if (ret) {
979                 PMD_INIT_LOG(ERR, "Failed to configure rq %d %d", qidx, ret);
980                 goto config_rq_error;
981         }
982         ret = nicvf_qset_cq_config(nic, qidx, rxq);
983         if (ret) {
984                 PMD_INIT_LOG(ERR, "Failed to configure cq %d %d", qidx, ret);
985                 goto config_cq_error;
986         }
987
988         dev->data->rx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STARTED;
989         return 0;
990
991 config_cq_error:
992         nicvf_qset_cq_reclaim(nic, qidx);
993 config_rq_error:
994         nicvf_qset_rq_reclaim(nic, qidx);
995         return ret;
996 }
997
998 static inline int
999 nicvf_stop_rx_queue(struct rte_eth_dev *dev, uint16_t qidx)
1000 {
1001         struct nicvf *nic = nicvf_pmd_priv(dev);
1002         struct nicvf_rxq *rxq;
1003         int ret, other_error;
1004
1005         if (dev->data->rx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STOPPED)
1006                 return 0;
1007
1008         ret = nicvf_qset_rq_reclaim(nic, qidx);
1009         if (ret)
1010                 PMD_INIT_LOG(ERR, "Failed to reclaim rq %d %d", qidx, ret);
1011
1012         other_error = ret;
1013         rxq = dev->data->rx_queues[qidx];
1014         nicvf_rx_queue_release_mbufs(rxq);
1015         nicvf_rx_queue_reset(rxq);
1016
1017         ret = nicvf_qset_cq_reclaim(nic, qidx);
1018         if (ret)
1019                 PMD_INIT_LOG(ERR, "Failed to reclaim cq %d %d", qidx, ret);
1020
1021         other_error |= ret;
1022         dev->data->rx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED;
1023         return other_error;
1024 }
1025
1026 static void
1027 nicvf_dev_rx_queue_release(void *rx_queue)
1028 {
1029         struct nicvf_rxq *rxq = rx_queue;
1030
1031         PMD_INIT_FUNC_TRACE();
1032
1033         if (rxq)
1034                 rte_free(rxq);
1035 }
1036
1037 static int
1038 nicvf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t qidx)
1039 {
1040         int ret;
1041
1042         ret = nicvf_start_rx_queue(dev, qidx);
1043         if (ret)
1044                 return ret;
1045
1046         ret = nicvf_configure_cpi(dev);
1047         if (ret)
1048                 return ret;
1049
1050         return nicvf_configure_rss_reta(dev);
1051 }
1052
1053 static int
1054 nicvf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx)
1055 {
1056         int ret;
1057
1058         ret = nicvf_stop_rx_queue(dev, qidx);
1059         ret |= nicvf_configure_cpi(dev);
1060         ret |= nicvf_configure_rss_reta(dev);
1061         return ret;
1062 }
1063
1064 static int
1065 nicvf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t qidx)
1066 {
1067         return nicvf_start_tx_queue(dev, qidx);
1068 }
1069
1070 static int
1071 nicvf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx)
1072 {
1073         return nicvf_stop_tx_queue(dev, qidx);
1074 }
1075
1076 static int
1077 nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
1078                          uint16_t nb_desc, unsigned int socket_id,
1079                          const struct rte_eth_rxconf *rx_conf,
1080                          struct rte_mempool *mp)
1081 {
1082         uint16_t rx_free_thresh;
1083         struct nicvf_rxq *rxq;
1084         struct nicvf *nic = nicvf_pmd_priv(dev);
1085
1086         PMD_INIT_FUNC_TRACE();
1087
1088         /* Socket id check */
1089         if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node)
1090                 PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d",
1091                 socket_id, nic->node);
1092
1093         /* Mempool memory should be contiguous */
1094         if (mp->nb_mem_chunks != 1) {
1095                 PMD_INIT_LOG(ERR, "Non contiguous mempool, check huge page sz");
1096                 return -EINVAL;
1097         }
1098
1099         /* Rx deferred start is not supported */
1100         if (rx_conf->rx_deferred_start) {
1101                 PMD_INIT_LOG(ERR, "Rx deferred start not supported");
1102                 return -EINVAL;
1103         }
1104
1105         /* Roundup nb_desc to available qsize and validate max number of desc */
1106         nb_desc = nicvf_qsize_cq_roundup(nb_desc);
1107         if (nb_desc == 0) {
1108                 PMD_INIT_LOG(ERR, "Value nb_desc beyond available hw cq qsize");
1109                 return -EINVAL;
1110         }
1111
1112         /* Check rx_free_thresh upper bound */
1113         rx_free_thresh = (uint16_t)((rx_conf->rx_free_thresh) ?
1114                                 rx_conf->rx_free_thresh :
1115                                 NICVF_DEFAULT_RX_FREE_THRESH);
1116         if (rx_free_thresh > NICVF_MAX_RX_FREE_THRESH ||
1117                 rx_free_thresh >= nb_desc * .75) {
1118                 PMD_INIT_LOG(ERR, "rx_free_thresh greater than expected %d",
1119                                 rx_free_thresh);
1120                 return -EINVAL;
1121         }
1122
1123         /* Free memory prior to re-allocation if needed */
1124         if (dev->data->rx_queues[qidx] != NULL) {
1125                 PMD_RX_LOG(DEBUG, "Freeing memory prior to re-allocation %d",
1126                                 qidx);
1127                 nicvf_dev_rx_queue_release(dev->data->rx_queues[qidx]);
1128                 dev->data->rx_queues[qidx] = NULL;
1129         }
1130
1131         /* Allocate rxq memory */
1132         rxq = rte_zmalloc_socket("ethdev rx queue", sizeof(struct nicvf_rxq),
1133                                         RTE_CACHE_LINE_SIZE, nic->node);
1134         if (rxq == NULL) {
1135                 PMD_INIT_LOG(ERR, "Failed to allocate rxq=%d", qidx);
1136                 return -ENOMEM;
1137         }
1138
1139         rxq->nic = nic;
1140         rxq->pool = mp;
1141         rxq->queue_id = qidx;
1142         rxq->port_id = dev->data->port_id;
1143         rxq->rx_free_thresh = rx_free_thresh;
1144         rxq->rx_drop_en = rx_conf->rx_drop_en;
1145         rxq->cq_status = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_STATUS;
1146         rxq->cq_door = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_DOOR;
1147         rxq->precharge_cnt = 0;
1148         rxq->rbptr_offset = NICVF_CQE_RBPTR_WORD;
1149
1150         /* Alloc completion queue */
1151         if (nicvf_qset_cq_alloc(nic, rxq, rxq->queue_id, nb_desc)) {
1152                 PMD_INIT_LOG(ERR, "failed to allocate cq %u", rxq->queue_id);
1153                 nicvf_dev_rx_queue_release(rxq);
1154                 return -ENOMEM;
1155         }
1156
1157         nicvf_rx_queue_reset(rxq);
1158
1159         PMD_RX_LOG(DEBUG, "[%d] rxq=%p pool=%s nb_desc=(%d/%d) phy=%" PRIx64,
1160                         qidx, rxq, mp->name, nb_desc,
1161                         rte_mempool_avail_count(mp), rxq->phys);
1162
1163         dev->data->rx_queues[qidx] = rxq;
1164         dev->data->rx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED;
1165         return 0;
1166 }
1167
1168 static void
1169 nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1170 {
1171         struct nicvf *nic = nicvf_pmd_priv(dev);
1172
1173         PMD_INIT_FUNC_TRACE();
1174
1175         dev_info->min_rx_bufsize = ETHER_MIN_MTU;
1176         dev_info->max_rx_pktlen = NIC_HW_MAX_FRS;
1177         dev_info->max_rx_queues = (uint16_t)MAX_RCV_QUEUES_PER_QS;
1178         dev_info->max_tx_queues = (uint16_t)MAX_SND_QUEUES_PER_QS;
1179         dev_info->max_mac_addrs = 1;
1180         dev_info->max_vfs = dev->pci_dev->max_vfs;
1181
1182         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
1183         dev_info->tx_offload_capa =
1184                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1185                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1186                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1187                 DEV_TX_OFFLOAD_TCP_TSO     |
1188                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
1189
1190         dev_info->reta_size = nic->rss_info.rss_size;
1191         dev_info->hash_key_size = RSS_HASH_KEY_BYTE_SIZE;
1192         dev_info->flow_type_rss_offloads = NICVF_RSS_OFFLOAD_PASS1;
1193         if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING)
1194                 dev_info->flow_type_rss_offloads |= NICVF_RSS_OFFLOAD_TUNNEL;
1195
1196         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1197                 .rx_free_thresh = NICVF_DEFAULT_RX_FREE_THRESH,
1198                 .rx_drop_en = 0,
1199         };
1200
1201         dev_info->default_txconf = (struct rte_eth_txconf) {
1202                 .tx_free_thresh = NICVF_DEFAULT_TX_FREE_THRESH,
1203                 .txq_flags =
1204                         ETH_TXQ_FLAGS_NOMULTSEGS  |
1205                         ETH_TXQ_FLAGS_NOREFCOUNT  |
1206                         ETH_TXQ_FLAGS_NOMULTMEMP  |
1207                         ETH_TXQ_FLAGS_NOVLANOFFL  |
1208                         ETH_TXQ_FLAGS_NOXSUMSCTP,
1209         };
1210 }
1211
1212 static nicvf_phys_addr_t
1213 rbdr_rte_mempool_get(void *opaque)
1214 {
1215         uint16_t qidx;
1216         uintptr_t mbuf;
1217         struct nicvf_rxq *rxq;
1218         struct nicvf *nic = nicvf_pmd_priv((struct rte_eth_dev *)opaque);
1219
1220         for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) {
1221                 rxq = nic->eth_dev->data->rx_queues[qidx];
1222                 /* Maintain equal buffer count across all pools */
1223                 if (rxq->precharge_cnt >= rxq->qlen_mask)
1224                         continue;
1225                 rxq->precharge_cnt++;
1226                 mbuf = (uintptr_t)rte_pktmbuf_alloc(rxq->pool);
1227                 if (mbuf)
1228                         return nicvf_mbuff_virt2phy(mbuf, rxq->mbuf_phys_off);
1229         }
1230         return 0;
1231 }
1232
1233 static int
1234 nicvf_dev_start(struct rte_eth_dev *dev)
1235 {
1236         int ret;
1237         uint16_t qidx;
1238         uint32_t buffsz = 0, rbdrsz = 0;
1239         uint32_t total_rxq_desc, nb_rbdr_desc, exp_buffs;
1240         uint64_t mbuf_phys_off = 0;
1241         struct nicvf_rxq *rxq;
1242         struct rte_pktmbuf_pool_private *mbp_priv;
1243         struct rte_mbuf *mbuf;
1244         struct nicvf *nic = nicvf_pmd_priv(dev);
1245         struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
1246         uint16_t mtu;
1247
1248         PMD_INIT_FUNC_TRACE();
1249
1250         /* Userspace process exited without proper shutdown in last run */
1251         if (nicvf_qset_rbdr_active(nic, 0))
1252                 nicvf_dev_stop(dev);
1253
1254         /*
1255          * Thunderx nicvf PMD can support more than one pool per port only when
1256          * 1) Data payload size is same across all the pools in given port
1257          * AND
1258          * 2) All mbuffs in the pools are from the same hugepage
1259          * AND
1260          * 3) Mbuff metadata size is same across all the pools in given port
1261          *
1262          * This is to support existing application that uses multiple pool/port.
1263          * But, the purpose of using multipool for QoS will not be addressed.
1264          *
1265          */
1266
1267         /* Validate RBDR buff size */
1268         for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) {
1269                 rxq = dev->data->rx_queues[qidx];
1270                 mbp_priv = rte_mempool_get_priv(rxq->pool);
1271                 buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM;
1272                 if (buffsz % 128) {
1273                         PMD_INIT_LOG(ERR, "rxbuf size must be multiply of 128");
1274                         return -EINVAL;
1275                 }
1276                 if (rbdrsz == 0)
1277                         rbdrsz = buffsz;
1278                 if (rbdrsz != buffsz) {
1279                         PMD_INIT_LOG(ERR, "buffsz not same, qid=%d (%d/%d)",
1280                                      qidx, rbdrsz, buffsz);
1281                         return -EINVAL;
1282                 }
1283         }
1284
1285         /* Validate mempool attributes */
1286         for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) {
1287                 rxq = dev->data->rx_queues[qidx];
1288                 rxq->mbuf_phys_off = nicvf_mempool_phy_offset(rxq->pool);
1289                 mbuf = rte_pktmbuf_alloc(rxq->pool);
1290                 if (mbuf == NULL) {
1291                         PMD_INIT_LOG(ERR, "Failed allocate mbuf qid=%d pool=%s",
1292                                      qidx, rxq->pool->name);
1293                         return -ENOMEM;
1294                 }
1295                 rxq->mbuf_phys_off -= nicvf_mbuff_meta_length(mbuf);
1296                 rxq->mbuf_phys_off -= RTE_PKTMBUF_HEADROOM;
1297                 rte_pktmbuf_free(mbuf);
1298
1299                 if (mbuf_phys_off == 0)
1300                         mbuf_phys_off = rxq->mbuf_phys_off;
1301                 if (mbuf_phys_off != rxq->mbuf_phys_off) {
1302                         PMD_INIT_LOG(ERR, "pool params not same,%s %" PRIx64,
1303                                      rxq->pool->name, mbuf_phys_off);
1304                         return -EINVAL;
1305                 }
1306         }
1307
1308         /* Check the level of buffers in the pool */
1309         total_rxq_desc = 0;
1310         for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) {
1311                 rxq = dev->data->rx_queues[qidx];
1312                 /* Count total numbers of rxq descs */
1313                 total_rxq_desc += rxq->qlen_mask + 1;
1314                 exp_buffs = RTE_MEMPOOL_CACHE_MAX_SIZE + rxq->rx_free_thresh;
1315                 exp_buffs *= nic->eth_dev->data->nb_rx_queues;
1316                 if (rte_mempool_avail_count(rxq->pool) < exp_buffs) {
1317                         PMD_INIT_LOG(ERR, "Buff shortage in pool=%s (%d/%d)",
1318                                      rxq->pool->name,
1319                                      rte_mempool_avail_count(rxq->pool),
1320                                      exp_buffs);
1321                         return -ENOENT;
1322                 }
1323         }
1324
1325         /* Check RBDR desc overflow */
1326         ret = nicvf_qsize_rbdr_roundup(total_rxq_desc);
1327         if (ret == 0) {
1328                 PMD_INIT_LOG(ERR, "Reached RBDR desc limit, reduce nr desc");
1329                 return -ENOMEM;
1330         }
1331
1332         /* Enable qset */
1333         ret = nicvf_qset_config(nic);
1334         if (ret) {
1335                 PMD_INIT_LOG(ERR, "Failed to enable qset %d", ret);
1336                 return ret;
1337         }
1338
1339         /* Allocate RBDR and RBDR ring desc */
1340         nb_rbdr_desc = nicvf_qsize_rbdr_roundup(total_rxq_desc);
1341         ret = nicvf_qset_rbdr_alloc(nic, nb_rbdr_desc, rbdrsz);
1342         if (ret) {
1343                 PMD_INIT_LOG(ERR, "Failed to allocate memory for rbdr alloc");
1344                 goto qset_reclaim;
1345         }
1346
1347         /* Enable and configure RBDR registers */
1348         ret = nicvf_qset_rbdr_config(nic, 0);
1349         if (ret) {
1350                 PMD_INIT_LOG(ERR, "Failed to configure rbdr %d", ret);
1351                 goto qset_rbdr_free;
1352         }
1353
1354         /* Fill rte_mempool buffers in RBDR pool and precharge it */
1355         ret = nicvf_qset_rbdr_precharge(nic, 0, rbdr_rte_mempool_get,
1356                                         dev, total_rxq_desc);
1357         if (ret) {
1358                 PMD_INIT_LOG(ERR, "Failed to fill rbdr %d", ret);
1359                 goto qset_rbdr_reclaim;
1360         }
1361
1362         PMD_DRV_LOG(INFO, "Filled %d out of %d entries in RBDR",
1363                      nic->rbdr->tail, nb_rbdr_desc);
1364
1365         /* Configure RX queues */
1366         for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) {
1367                 ret = nicvf_start_rx_queue(dev, qidx);
1368                 if (ret)
1369                         goto start_rxq_error;
1370         }
1371
1372         /* Configure VLAN Strip */
1373         nicvf_vlan_hw_strip(nic, dev->data->dev_conf.rxmode.hw_vlan_strip);
1374
1375         /* Configure TX queues */
1376         for (qidx = 0; qidx < nic->eth_dev->data->nb_tx_queues; qidx++) {
1377                 ret = nicvf_start_tx_queue(dev, qidx);
1378                 if (ret)
1379                         goto start_txq_error;
1380         }
1381
1382         /* Configure CPI algorithm */
1383         ret = nicvf_configure_cpi(dev);
1384         if (ret)
1385                 goto start_txq_error;
1386
1387         /* Configure RSS */
1388         ret = nicvf_configure_rss(dev);
1389         if (ret)
1390                 goto qset_rss_error;
1391
1392         /* Configure loopback */
1393         ret = nicvf_loopback_config(nic, dev->data->dev_conf.lpbk_mode);
1394         if (ret) {
1395                 PMD_INIT_LOG(ERR, "Failed to configure loopback %d", ret);
1396                 goto qset_rss_error;
1397         }
1398
1399         /* Reset all statistics counters attached to this port */
1400         ret = nicvf_mbox_reset_stat_counters(nic, 0x3FFF, 0x1F, 0xFFFF, 0xFFFF);
1401         if (ret) {
1402                 PMD_INIT_LOG(ERR, "Failed to reset stat counters %d", ret);
1403                 goto qset_rss_error;
1404         }
1405
1406         /* Setup scatter mode if needed by jumbo */
1407         if (dev->data->dev_conf.rxmode.max_rx_pkt_len +
1408                                             2 * VLAN_TAG_SIZE > buffsz)
1409                 dev->data->scattered_rx = 1;
1410         if (rx_conf->enable_scatter)
1411                 dev->data->scattered_rx = 1;
1412
1413         /* Setup MTU based on max_rx_pkt_len or default */
1414         mtu = dev->data->dev_conf.rxmode.jumbo_frame ?
1415                 dev->data->dev_conf.rxmode.max_rx_pkt_len
1416                         -  ETHER_HDR_LEN - ETHER_CRC_LEN
1417                 : ETHER_MTU;
1418
1419         if (nicvf_dev_set_mtu(dev, mtu)) {
1420                 PMD_INIT_LOG(ERR, "Failed to set default mtu size");
1421                 return -EBUSY;
1422         }
1423
1424         /* Configure callbacks based on scatter mode */
1425         nicvf_set_tx_function(dev);
1426         nicvf_set_rx_function(dev);
1427
1428         /* Done; Let PF make the BGX's RX and TX switches to ON position */
1429         nicvf_mbox_cfg_done(nic);
1430         return 0;
1431
1432 qset_rss_error:
1433         nicvf_rss_term(nic);
1434 start_txq_error:
1435         for (qidx = 0; qidx < nic->eth_dev->data->nb_tx_queues; qidx++)
1436                 nicvf_stop_tx_queue(dev, qidx);
1437 start_rxq_error:
1438         for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++)
1439                 nicvf_stop_rx_queue(dev, qidx);
1440 qset_rbdr_reclaim:
1441         nicvf_qset_rbdr_reclaim(nic, 0);
1442         nicvf_rbdr_release_mbufs(nic);
1443 qset_rbdr_free:
1444         if (nic->rbdr) {
1445                 rte_free(nic->rbdr);
1446                 nic->rbdr = NULL;
1447         }
1448 qset_reclaim:
1449         nicvf_qset_reclaim(nic);
1450         return ret;
1451 }
1452
1453 static void
1454 nicvf_dev_stop(struct rte_eth_dev *dev)
1455 {
1456         int ret;
1457         uint16_t qidx;
1458         struct nicvf *nic = nicvf_pmd_priv(dev);
1459
1460         PMD_INIT_FUNC_TRACE();
1461
1462         /* Let PF make the BGX's RX and TX switches to OFF position */
1463         nicvf_mbox_shutdown(nic);
1464
1465         /* Disable loopback */
1466         ret = nicvf_loopback_config(nic, 0);
1467         if (ret)
1468                 PMD_INIT_LOG(ERR, "Failed to disable loopback %d", ret);
1469
1470         /* Disable VLAN Strip */
1471         nicvf_vlan_hw_strip(nic, 0);
1472
1473         /* Reclaim sq */
1474         for (qidx = 0; qidx < dev->data->nb_tx_queues; qidx++)
1475                 nicvf_stop_tx_queue(dev, qidx);
1476
1477         /* Reclaim rq */
1478         for (qidx = 0; qidx < dev->data->nb_rx_queues; qidx++)
1479                 nicvf_stop_rx_queue(dev, qidx);
1480
1481         /* Reclaim RBDR */
1482         ret = nicvf_qset_rbdr_reclaim(nic, 0);
1483         if (ret)
1484                 PMD_INIT_LOG(ERR, "Failed to reclaim RBDR %d", ret);
1485
1486         /* Move all charged buffers in RBDR back to pool */
1487         if (nic->rbdr != NULL)
1488                 nicvf_rbdr_release_mbufs(nic);
1489
1490         /* Reclaim CPI configuration */
1491         if (!nic->sqs_mode) {
1492                 ret = nicvf_mbox_config_cpi(nic, 0);
1493                 if (ret)
1494                         PMD_INIT_LOG(ERR, "Failed to reclaim CPI config");
1495         }
1496
1497         /* Disable qset */
1498         ret = nicvf_qset_config(nic);
1499         if (ret)
1500                 PMD_INIT_LOG(ERR, "Failed to disable qset %d", ret);
1501
1502         /* Disable all interrupts */
1503         nicvf_disable_all_interrupts(nic);
1504
1505         /* Free RBDR SW structure */
1506         if (nic->rbdr) {
1507                 rte_free(nic->rbdr);
1508                 nic->rbdr = NULL;
1509         }
1510 }
1511
1512 static void
1513 nicvf_dev_close(struct rte_eth_dev *dev)
1514 {
1515         struct nicvf *nic = nicvf_pmd_priv(dev);
1516
1517         PMD_INIT_FUNC_TRACE();
1518
1519         nicvf_dev_stop(dev);
1520         nicvf_periodic_alarm_stop(nic);
1521 }
1522
1523 static int
1524 nicvf_dev_configure(struct rte_eth_dev *dev)
1525 {
1526         struct rte_eth_conf *conf = &dev->data->dev_conf;
1527         struct rte_eth_rxmode *rxmode = &conf->rxmode;
1528         struct rte_eth_txmode *txmode = &conf->txmode;
1529         struct nicvf *nic = nicvf_pmd_priv(dev);
1530
1531         PMD_INIT_FUNC_TRACE();
1532
1533         if (!rte_eal_has_hugepages()) {
1534                 PMD_INIT_LOG(INFO, "Huge page is not configured");
1535                 return -EINVAL;
1536         }
1537
1538         if (txmode->mq_mode) {
1539                 PMD_INIT_LOG(INFO, "Tx mq_mode DCB or VMDq not supported");
1540                 return -EINVAL;
1541         }
1542
1543         if (rxmode->mq_mode != ETH_MQ_RX_NONE &&
1544                 rxmode->mq_mode != ETH_MQ_RX_RSS) {
1545                 PMD_INIT_LOG(INFO, "Unsupported rx qmode %d", rxmode->mq_mode);
1546                 return -EINVAL;
1547         }
1548
1549         if (!rxmode->hw_strip_crc) {
1550                 PMD_INIT_LOG(NOTICE, "Can't disable hw crc strip");
1551                 rxmode->hw_strip_crc = 1;
1552         }
1553
1554         if (rxmode->hw_ip_checksum) {
1555                 PMD_INIT_LOG(NOTICE, "Rxcksum not supported");
1556                 rxmode->hw_ip_checksum = 0;
1557         }
1558
1559         if (rxmode->split_hdr_size) {
1560                 PMD_INIT_LOG(INFO, "Rxmode does not support split header");
1561                 return -EINVAL;
1562         }
1563
1564         if (rxmode->hw_vlan_filter) {
1565                 PMD_INIT_LOG(INFO, "VLAN filter not supported");
1566                 return -EINVAL;
1567         }
1568
1569         if (rxmode->hw_vlan_extend) {
1570                 PMD_INIT_LOG(INFO, "VLAN extended not supported");
1571                 return -EINVAL;
1572         }
1573
1574         if (rxmode->enable_lro) {
1575                 PMD_INIT_LOG(INFO, "LRO not supported");
1576                 return -EINVAL;
1577         }
1578
1579         if (conf->link_speeds & ETH_LINK_SPEED_FIXED) {
1580                 PMD_INIT_LOG(INFO, "Setting link speed/duplex not supported");
1581                 return -EINVAL;
1582         }
1583
1584         if (conf->dcb_capability_en) {
1585                 PMD_INIT_LOG(INFO, "DCB enable not supported");
1586                 return -EINVAL;
1587         }
1588
1589         if (conf->fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1590                 PMD_INIT_LOG(INFO, "Flow director not supported");
1591                 return -EINVAL;
1592         }
1593
1594         PMD_INIT_LOG(DEBUG, "Configured ethdev port%d hwcap=0x%" PRIx64,
1595                 dev->data->port_id, nicvf_hw_cap(nic));
1596
1597         return 0;
1598 }
1599
1600 /* Initialize and register driver with DPDK Application */
1601 static const struct eth_dev_ops nicvf_eth_dev_ops = {
1602         .dev_configure            = nicvf_dev_configure,
1603         .dev_start                = nicvf_dev_start,
1604         .dev_stop                 = nicvf_dev_stop,
1605         .link_update              = nicvf_dev_link_update,
1606         .dev_close                = nicvf_dev_close,
1607         .stats_get                = nicvf_dev_stats_get,
1608         .stats_reset              = nicvf_dev_stats_reset,
1609         .promiscuous_enable       = nicvf_dev_promisc_enable,
1610         .dev_infos_get            = nicvf_dev_info_get,
1611         .dev_supported_ptypes_get = nicvf_dev_supported_ptypes_get,
1612         .mtu_set                  = nicvf_dev_set_mtu,
1613         .reta_update              = nicvf_dev_reta_update,
1614         .reta_query               = nicvf_dev_reta_query,
1615         .rss_hash_update          = nicvf_dev_rss_hash_update,
1616         .rss_hash_conf_get        = nicvf_dev_rss_hash_conf_get,
1617         .rx_queue_start           = nicvf_dev_rx_queue_start,
1618         .rx_queue_stop            = nicvf_dev_rx_queue_stop,
1619         .tx_queue_start           = nicvf_dev_tx_queue_start,
1620         .tx_queue_stop            = nicvf_dev_tx_queue_stop,
1621         .rx_queue_setup           = nicvf_dev_rx_queue_setup,
1622         .rx_queue_release         = nicvf_dev_rx_queue_release,
1623         .rx_queue_count           = nicvf_dev_rx_queue_count,
1624         .tx_queue_setup           = nicvf_dev_tx_queue_setup,
1625         .tx_queue_release         = nicvf_dev_tx_queue_release,
1626         .get_reg_length           = nicvf_dev_get_reg_length,
1627         .get_reg                  = nicvf_dev_get_regs,
1628 };
1629
1630 static int
1631 nicvf_eth_dev_init(struct rte_eth_dev *eth_dev)
1632 {
1633         int ret;
1634         struct rte_pci_device *pci_dev;
1635         struct nicvf *nic = nicvf_pmd_priv(eth_dev);
1636
1637         PMD_INIT_FUNC_TRACE();
1638
1639         eth_dev->dev_ops = &nicvf_eth_dev_ops;
1640
1641         /* For secondary processes, the primary has done all the work */
1642         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
1643                 /* Setup callbacks for secondary process */
1644                 nicvf_set_tx_function(eth_dev);
1645                 nicvf_set_rx_function(eth_dev);
1646                 return 0;
1647         }
1648
1649         pci_dev = eth_dev->pci_dev;
1650         rte_eth_copy_pci_info(eth_dev, pci_dev);
1651
1652         nic->device_id = pci_dev->id.device_id;
1653         nic->vendor_id = pci_dev->id.vendor_id;
1654         nic->subsystem_device_id = pci_dev->id.subsystem_device_id;
1655         nic->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
1656         nic->eth_dev = eth_dev;
1657
1658         PMD_INIT_LOG(DEBUG, "nicvf: device (%x:%x) %u:%u:%u:%u",
1659                         pci_dev->id.vendor_id, pci_dev->id.device_id,
1660                         pci_dev->addr.domain, pci_dev->addr.bus,
1661                         pci_dev->addr.devid, pci_dev->addr.function);
1662
1663         nic->reg_base = (uintptr_t)pci_dev->mem_resource[0].addr;
1664         if (!nic->reg_base) {
1665                 PMD_INIT_LOG(ERR, "Failed to map BAR0");
1666                 ret = -ENODEV;
1667                 goto fail;
1668         }
1669
1670         nicvf_disable_all_interrupts(nic);
1671
1672         ret = nicvf_periodic_alarm_start(nic);
1673         if (ret) {
1674                 PMD_INIT_LOG(ERR, "Failed to start period alarm");
1675                 goto fail;
1676         }
1677
1678         ret = nicvf_mbox_check_pf_ready(nic);
1679         if (ret) {
1680                 PMD_INIT_LOG(ERR, "Failed to get ready message from PF");
1681                 goto alarm_fail;
1682         } else {
1683                 PMD_INIT_LOG(INFO,
1684                         "node=%d vf=%d mode=%s sqs=%s loopback_supported=%s",
1685                         nic->node, nic->vf_id,
1686                         nic->tns_mode == NIC_TNS_MODE ? "tns" : "tns-bypass",
1687                         nic->sqs_mode ? "true" : "false",
1688                         nic->loopback_supported ? "true" : "false"
1689                         );
1690         }
1691
1692         if (nic->sqs_mode) {
1693                 PMD_INIT_LOG(INFO, "Unsupported SQS VF detected, Detaching...");
1694                 /* Detach port by returning Positive error number */
1695                 ret = ENOTSUP;
1696                 goto alarm_fail;
1697         }
1698
1699         eth_dev->data->mac_addrs = rte_zmalloc("mac_addr", ETHER_ADDR_LEN, 0);
1700         if (eth_dev->data->mac_addrs == NULL) {
1701                 PMD_INIT_LOG(ERR, "Failed to allocate memory for mac addr");
1702                 ret = -ENOMEM;
1703                 goto alarm_fail;
1704         }
1705         if (is_zero_ether_addr((struct ether_addr *)nic->mac_addr))
1706                 eth_random_addr(&nic->mac_addr[0]);
1707
1708         ether_addr_copy((struct ether_addr *)nic->mac_addr,
1709                         &eth_dev->data->mac_addrs[0]);
1710
1711         ret = nicvf_mbox_set_mac_addr(nic, nic->mac_addr);
1712         if (ret) {
1713                 PMD_INIT_LOG(ERR, "Failed to set mac addr");
1714                 goto malloc_fail;
1715         }
1716
1717         ret = nicvf_base_init(nic);
1718         if (ret) {
1719                 PMD_INIT_LOG(ERR, "Failed to execute nicvf_base_init");
1720                 goto malloc_fail;
1721         }
1722
1723         ret = nicvf_mbox_get_rss_size(nic);
1724         if (ret) {
1725                 PMD_INIT_LOG(ERR, "Failed to get rss table size");
1726                 goto malloc_fail;
1727         }
1728
1729         PMD_INIT_LOG(INFO, "Port %d (%x:%x) mac=%02x:%02x:%02x:%02x:%02x:%02x",
1730                 eth_dev->data->port_id, nic->vendor_id, nic->device_id,
1731                 nic->mac_addr[0], nic->mac_addr[1], nic->mac_addr[2],
1732                 nic->mac_addr[3], nic->mac_addr[4], nic->mac_addr[5]);
1733
1734         return 0;
1735
1736 malloc_fail:
1737         rte_free(eth_dev->data->mac_addrs);
1738 alarm_fail:
1739         nicvf_periodic_alarm_stop(nic);
1740 fail:
1741         return ret;
1742 }
1743
1744 static const struct rte_pci_id pci_id_nicvf_map[] = {
1745         {
1746                 .class_id = RTE_CLASS_ANY_ID,
1747                 .vendor_id = PCI_VENDOR_ID_CAVIUM,
1748                 .device_id = PCI_DEVICE_ID_THUNDERX_PASS1_NICVF,
1749                 .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
1750                 .subsystem_device_id = PCI_SUB_DEVICE_ID_THUNDERX_PASS1_NICVF,
1751         },
1752         {
1753                 .class_id = RTE_CLASS_ANY_ID,
1754                 .vendor_id = PCI_VENDOR_ID_CAVIUM,
1755                 .device_id = PCI_DEVICE_ID_THUNDERX_PASS2_NICVF,
1756                 .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
1757                 .subsystem_device_id = PCI_SUB_DEVICE_ID_THUNDERX_PASS2_NICVF,
1758         },
1759         {
1760                 .vendor_id = 0,
1761         },
1762 };
1763
1764 static struct eth_driver rte_nicvf_pmd = {
1765         .pci_drv = {
1766                 .name = "rte_nicvf_pmd",
1767                 .id_table = pci_id_nicvf_map,
1768                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
1769         },
1770         .eth_dev_init = nicvf_eth_dev_init,
1771         .dev_private_size = sizeof(struct nicvf),
1772 };
1773
1774 static int
1775 rte_nicvf_pmd_init(const char *name __rte_unused, const char *para __rte_unused)
1776 {
1777         PMD_INIT_FUNC_TRACE();
1778         PMD_INIT_LOG(INFO, "librte_pmd_thunderx nicvf version %s",
1779                         THUNDERX_NICVF_PMD_VERSION);
1780
1781         rte_eth_driver_register(&rte_nicvf_pmd);
1782         return 0;
1783 }
1784
1785 static struct rte_driver rte_nicvf_driver = {
1786         .name = "nicvf_driver",
1787         .type = PMD_PDEV,
1788         .init = rte_nicvf_pmd_init,
1789 };
1790
1791 PMD_REGISTER_DRIVER(rte_nicvf_driver, thunderx_nicvf);
1792 DRIVER_REGISTER_PCI_TABLE(thunderx_nicvf, pci_id_nicvf_map);