mbuf: replace data pointer by an offset
[dpdk.git] / lib / librte_pmd_vmxnet3 / vmxnet3_ethdev.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 #include <sys/queue.h>
35 #include <stdio.h>
36 #include <errno.h>
37 #include <stdint.h>
38 #include <string.h>
39 #include <unistd.h>
40 #include <stdarg.h>
41 #include <fcntl.h>
42 #include <inttypes.h>
43 #include <rte_byteorder.h>
44 #include <rte_common.h>
45 #include <rte_cycles.h>
46
47 #include <rte_interrupts.h>
48 #include <rte_log.h>
49 #include <rte_debug.h>
50 #include <rte_pci.h>
51 #include <rte_atomic.h>
52 #include <rte_branch_prediction.h>
53 #include <rte_memory.h>
54 #include <rte_memzone.h>
55 #include <rte_tailq.h>
56 #include <rte_eal.h>
57 #include <rte_alarm.h>
58 #include <rte_ether.h>
59 #include <rte_ethdev.h>
60 #include <rte_atomic.h>
61 #include <rte_string_fns.h>
62 #include <rte_malloc.h>
63 #include <rte_dev.h>
64
65 #include "vmxnet3/vmxnet3_defs.h"
66
67 #include "vmxnet3_ring.h"
68 #include "vmxnet3_logs.h"
69 #include "vmxnet3_ethdev.h"
70
71 #define PROCESS_SYS_EVENTS 0
72
73 static int eth_vmxnet3_dev_init(struct eth_driver *eth_drv,
74                 struct rte_eth_dev *eth_dev);
75 static int vmxnet3_dev_configure(struct rte_eth_dev *dev);
76 static int vmxnet3_dev_start(struct rte_eth_dev *dev);
77 static void vmxnet3_dev_stop(struct rte_eth_dev *dev);
78 static void vmxnet3_dev_close(struct rte_eth_dev *dev);
79 static void vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set);
80 static void vmxnet3_dev_promiscuous_enable(struct rte_eth_dev *dev);
81 static void vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev);
82 static void vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev);
83 static void vmxnet3_dev_allmulticast_disable(struct rte_eth_dev *dev);
84 static int vmxnet3_dev_link_update(struct rte_eth_dev *dev,
85                                 int wait_to_complete);
86 static void vmxnet3_dev_stats_get(struct rte_eth_dev *dev,
87                                 struct rte_eth_stats *stats);
88 static void vmxnet3_dev_info_get(struct rte_eth_dev *dev,
89                                 struct rte_eth_dev_info *dev_info);
90 #if PROCESS_SYS_EVENTS == 1
91 static void vmxnet3_process_events(struct vmxnet3_hw *);
92 #endif
93 /*
94  * The set of PCI devices this driver supports
95  */
96 static struct rte_pci_id pci_id_vmxnet3_map[] = {
97
98 #define RTE_PCI_DEV_ID_DECL_VMXNET3(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
99 #include "rte_pci_dev_ids.h"
100
101 { .vendor_id = 0, /* sentinel */ },
102 };
103
104 static struct eth_dev_ops vmxnet3_eth_dev_ops = {
105         .dev_configure        = vmxnet3_dev_configure,
106         .dev_start            = vmxnet3_dev_start,
107         .dev_stop             = vmxnet3_dev_stop,
108         .dev_close            = vmxnet3_dev_close,
109         .promiscuous_enable   = vmxnet3_dev_promiscuous_enable,
110         .promiscuous_disable  = vmxnet3_dev_promiscuous_disable,
111         .allmulticast_enable  = vmxnet3_dev_allmulticast_enable,
112         .allmulticast_disable = vmxnet3_dev_allmulticast_disable,
113         .link_update          = vmxnet3_dev_link_update,
114         .stats_get            = vmxnet3_dev_stats_get,
115         .dev_infos_get        = vmxnet3_dev_info_get,
116         .rx_queue_setup       = vmxnet3_dev_rx_queue_setup,
117         .rx_queue_release     = vmxnet3_dev_rx_queue_release,
118         .tx_queue_setup       = vmxnet3_dev_tx_queue_setup,
119         .tx_queue_release     = vmxnet3_dev_tx_queue_release,
120 };
121
122 static const struct rte_memzone *
123 gpa_zone_reserve(struct rte_eth_dev *dev, uint32_t size,
124                 const char *post_string, int socket_id, uint16_t align)
125 {
126         char z_name[RTE_MEMZONE_NAMESIZE];
127         const struct rte_memzone *mz;
128
129         snprintf(z_name, sizeof(z_name), "%s_%d_%s",
130                                         dev->driver->pci_drv.name, dev->data->port_id, post_string);
131
132         mz = rte_memzone_lookup(z_name);
133         if (mz)
134                 return mz;
135
136         return rte_memzone_reserve_aligned(z_name, size,
137                         socket_id, 0, align);
138 }
139
140 /**
141  * Atomically reads the link status information from global
142  * structure rte_eth_dev.
143  *
144  * @param dev
145  *   - Pointer to the structure rte_eth_dev to read from.
146  *   - Pointer to the buffer to be saved with the link status.
147  *
148  * @return
149  *   - On success, zero.
150  *   - On failure, negative value.
151  */
152 static inline int
153 rte_vmxnet3_dev_atomic_write_link_status(struct rte_eth_dev *dev,
154                                 struct rte_eth_link *link)
155 {
156         struct rte_eth_link *dst = &(dev->data->dev_link);
157         struct rte_eth_link *src = link;
158
159         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
160                                         *(uint64_t *)src) == 0)
161                 return -1;
162
163         return 0;
164 }
165
166 /*
167  * This function is based on vmxnet3_disable_intr()
168  */
169 static void
170 vmxnet3_disable_intr(struct vmxnet3_hw *hw)
171 {
172         int i;
173
174         PMD_INIT_FUNC_TRACE();
175
176         hw->shared->devRead.intrConf.intrCtrl |= VMXNET3_IC_DISABLE_ALL;
177         for (i = 0; i < VMXNET3_MAX_INTRS; i++)
178                         VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + i * 8, 1);
179 }
180
181 /*
182  * It returns 0 on success.
183  */
184 static int
185 eth_vmxnet3_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
186                      struct rte_eth_dev *eth_dev)
187 {
188         struct rte_pci_device *pci_dev;
189         struct vmxnet3_hw *hw = eth_dev->data->dev_private;
190         uint32_t mac_hi, mac_lo, ver;
191
192         PMD_INIT_FUNC_TRACE();
193
194         eth_dev->dev_ops = &vmxnet3_eth_dev_ops;
195         eth_dev->rx_pkt_burst = &vmxnet3_recv_pkts;
196         eth_dev->tx_pkt_burst = &vmxnet3_xmit_pkts;
197         pci_dev = eth_dev->pci_dev;
198
199         /*
200          * for secondary processes, we don't initialise any further as primary
201          * has already done this work.
202          */
203         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
204                 return 0;
205
206         /* Vendor and Device ID need to be set before init of shared code */
207         hw->device_id = pci_dev->id.device_id;
208         hw->vendor_id = pci_dev->id.vendor_id;
209         hw->hw_addr0 = (void *)pci_dev->mem_resource[0].addr;
210         hw->hw_addr1 = (void *)pci_dev->mem_resource[1].addr;
211
212         hw->num_rx_queues = 1;
213         hw->num_tx_queues = 1;
214         hw->cur_mtu = ETHER_MTU;
215         hw->bufs_per_pkt = 1;
216
217         /* Check h/w version compatibility with driver. */
218         ver = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_VRRS);
219         PMD_INIT_LOG(DEBUG, "Harware version : %d", ver);
220         if (ver & 0x1)
221                 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 1);
222         else {
223                 PMD_INIT_LOG(ERR, "Uncompatiable h/w version, should be 0x1");
224                 return -EIO;
225         }
226
227         /* Check UPT version compatibility with driver. */
228         ver = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_UVRS);
229         PMD_INIT_LOG(DEBUG, "UPT harware version : %d", ver);
230         if (ver & 0x1)
231                 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_UVRS, 1);
232         else {
233                 PMD_INIT_LOG(ERR, "Incompatiable UPT version.");
234                 return -EIO;
235         }
236
237         /* Getting MAC Address */
238         mac_lo = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACL);
239         mac_hi = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACH);
240         memcpy(hw->perm_addr  , &mac_lo, 4);
241         memcpy(hw->perm_addr+4, &mac_hi, 2);
242
243         /* Allocate memory for storing MAC addresses */
244         eth_dev->data->mac_addrs = rte_zmalloc("vmxnet3", ETHER_ADDR_LEN *
245                                                VMXNET3_MAX_MAC_ADDRS, 0);
246         if (eth_dev->data->mac_addrs == NULL) {
247                 PMD_INIT_LOG(ERR,
248                              "Failed to allocate %d bytes needed to store MAC addresses",
249                              ETHER_ADDR_LEN * VMXNET3_MAX_MAC_ADDRS);
250                 return -ENOMEM;
251         }
252         /* Copy the permanent MAC address */
253         ether_addr_copy((struct ether_addr *) hw->perm_addr,
254                         &eth_dev->data->mac_addrs[0]);
255
256         PMD_INIT_LOG(DEBUG, "MAC Address : %02x:%02x:%02x:%02x:%02x:%02x",
257                      hw->perm_addr[0], hw->perm_addr[1], hw->perm_addr[2],
258                      hw->perm_addr[3], hw->perm_addr[4], hw->perm_addr[5]);
259
260         /* Put device in Quiesce Mode */
261         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
262
263         return 0;
264 }
265
266 static struct eth_driver rte_vmxnet3_pmd = {
267         {
268                 .name = "rte_vmxnet3_pmd",
269                 .id_table = pci_id_vmxnet3_map,
270                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
271         },
272         .eth_dev_init = eth_vmxnet3_dev_init,
273         .dev_private_size = sizeof(struct vmxnet3_hw),
274 };
275
276 /*
277  * Driver initialization routine.
278  * Invoked once at EAL init time.
279  * Register itself as the [Poll Mode] Driver of Virtual PCI VMXNET3 devices.
280  */
281 static int
282 rte_vmxnet3_pmd_init(const char *name __rte_unused, const char *param __rte_unused)
283 {
284         PMD_INIT_FUNC_TRACE();
285
286         rte_eth_driver_register(&rte_vmxnet3_pmd);
287         return 0;
288 }
289
290 static int
291 vmxnet3_dev_configure(struct rte_eth_dev *dev)
292 {
293         const struct rte_memzone *mz;
294         struct vmxnet3_hw *hw = dev->data->dev_private;
295         size_t size;
296
297         PMD_INIT_FUNC_TRACE();
298
299         if (dev->data->nb_rx_queues > UINT8_MAX ||
300             dev->data->nb_tx_queues > UINT8_MAX)
301                 return -EINVAL;
302
303         size = dev->data->nb_rx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
304                 dev->data->nb_tx_queues * sizeof(struct Vmxnet3_RxQueueDesc);
305
306         if (size > UINT16_MAX)
307                 return -EINVAL;
308
309         hw->num_rx_queues = (uint8_t)dev->data->nb_rx_queues;
310         hw->num_tx_queues = (uint8_t)dev->data->nb_tx_queues;
311
312         /*
313          * Allocate a memzone for Vmxnet3_DriverShared - Vmxnet3_DSDevRead
314          * on current socket
315          */
316         mz = gpa_zone_reserve(dev, sizeof(struct Vmxnet3_DriverShared),
317                               "shared", rte_socket_id(), 8);
318
319         if (mz == NULL) {
320                 PMD_INIT_LOG(ERR, "ERROR: Creating shared zone");
321                 return -ENOMEM;
322         }
323         memset(mz->addr, 0, mz->len);
324
325         hw->shared = mz->addr;
326         hw->sharedPA = mz->phys_addr;
327
328         /*
329          * Allocate a memzone for Vmxnet3_RxQueueDesc - Vmxnet3_TxQueueDesc
330          * on current socket
331          */
332         mz = gpa_zone_reserve(dev, size, "queuedesc",
333                               rte_socket_id(), VMXNET3_QUEUE_DESC_ALIGN);
334         if (mz == NULL) {
335                 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
336                 return -ENOMEM;
337         }
338         memset(mz->addr, 0, mz->len);
339
340         hw->tqd_start = (Vmxnet3_TxQueueDesc *)mz->addr;
341         hw->rqd_start = (Vmxnet3_RxQueueDesc *)(hw->tqd_start + hw->num_tx_queues);
342
343         hw->queueDescPA = mz->phys_addr;
344         hw->queue_desc_len = (uint16_t)size;
345
346         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
347
348                 /* Allocate memory structure for UPT1_RSSConf and configure */
349                 mz = gpa_zone_reserve(dev, sizeof(struct VMXNET3_RSSConf), "rss_conf",
350                                       rte_socket_id(), CACHE_LINE_SIZE);
351                 if (mz == NULL) {
352                         PMD_INIT_LOG(ERR,
353                                      "ERROR: Creating rss_conf structure zone");
354                         return -ENOMEM;
355                 }
356                 memset(mz->addr, 0, mz->len);
357
358                 hw->rss_conf = mz->addr;
359                 hw->rss_confPA = mz->phys_addr;
360         }
361
362         return 0;
363 }
364
365 static int
366 vmxnet3_setup_driver_shared(struct rte_eth_dev *dev)
367 {
368         struct rte_eth_conf port_conf = dev->data->dev_conf;
369         struct vmxnet3_hw *hw = dev->data->dev_private;
370         Vmxnet3_DriverShared *shared = hw->shared;
371         Vmxnet3_DSDevRead *devRead = &shared->devRead;
372         uint32_t *mac_ptr;
373         uint32_t val, i;
374         int ret;
375
376         shared->magic = VMXNET3_REV1_MAGIC;
377         devRead->misc.driverInfo.version = VMXNET3_DRIVER_VERSION_NUM;
378
379         /* Setting up Guest OS information */
380         devRead->misc.driverInfo.gos.gosBits   = sizeof(void *) == 4 ?
381                 VMXNET3_GOS_BITS_32 :
382                 VMXNET3_GOS_BITS_64;
383         devRead->misc.driverInfo.gos.gosType   = VMXNET3_GOS_TYPE_LINUX;
384         devRead->misc.driverInfo.vmxnet3RevSpt = 1;
385         devRead->misc.driverInfo.uptVerSpt     = 1;
386
387         devRead->misc.queueDescPA  = hw->queueDescPA;
388         devRead->misc.queueDescLen = hw->queue_desc_len;
389         devRead->misc.mtu          = hw->cur_mtu;
390         devRead->misc.numTxQueues  = hw->num_tx_queues;
391         devRead->misc.numRxQueues  = hw->num_rx_queues;
392
393         /*
394          * Set number of interrupts to 1
395          * PMD disables all the interrupts but this is MUST to activate device
396          * It needs at least one interrupt for link events to handle
397          * So we'll disable it later after device activation if needed
398          */
399         devRead->intrConf.numIntrs = 1;
400         devRead->intrConf.intrCtrl |= VMXNET3_IC_DISABLE_ALL;
401
402         for (i = 0; i < hw->num_tx_queues; i++) {
403                 Vmxnet3_TxQueueDesc *tqd = &hw->tqd_start[i];
404                 vmxnet3_tx_queue_t *txq   = dev->data->tx_queues[i];
405
406                 tqd->ctrl.txNumDeferred  = 0;
407                 tqd->ctrl.txThreshold    = 1;
408                 tqd->conf.txRingBasePA   = txq->cmd_ring.basePA;
409                 tqd->conf.compRingBasePA = txq->comp_ring.basePA;
410
411                 tqd->conf.txRingSize   = txq->cmd_ring.size;
412                 tqd->conf.compRingSize = txq->comp_ring.size;
413                 tqd->conf.intrIdx      = txq->comp_ring.intr_idx;
414                 tqd->status.stopped    = TRUE;
415                 tqd->status.error      = 0;
416                 memset(&tqd->stats, 0, sizeof(tqd->stats));
417         }
418
419         for (i = 0; i < hw->num_rx_queues; i++) {
420                 Vmxnet3_RxQueueDesc *rqd  = &hw->rqd_start[i];
421                 vmxnet3_rx_queue_t *rxq    = dev->data->rx_queues[i];
422
423                 rqd->conf.rxRingBasePA[0] = rxq->cmd_ring[0].basePA;
424                 rqd->conf.rxRingBasePA[1] = rxq->cmd_ring[1].basePA;
425                 rqd->conf.compRingBasePA  = rxq->comp_ring.basePA;
426
427                 rqd->conf.rxRingSize[0]   = rxq->cmd_ring[0].size;
428                 rqd->conf.rxRingSize[1]   = rxq->cmd_ring[1].size;
429                 rqd->conf.compRingSize    = rxq->comp_ring.size;
430                 rqd->conf.intrIdx         = rxq->comp_ring.intr_idx;
431                 rqd->status.stopped       = TRUE;
432                 rqd->status.error         = 0;
433                 memset(&rqd->stats, 0, sizeof(rqd->stats));
434         }
435
436         /* RxMode set to 0 of VMXNET3_RXM_xxx */
437         devRead->rxFilterConf.rxMode = 0;
438
439         /* Setting up feature flags */
440         if (dev->data->dev_conf.rxmode.hw_ip_checksum)
441                 devRead->misc.uptFeatures |= VMXNET3_F_RXCSUM;
442
443         if (dev->data->dev_conf.rxmode.hw_vlan_strip)
444                 devRead->misc.uptFeatures |= VMXNET3_F_RXVLAN;
445
446         if (port_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
447                 ret = vmxnet3_rss_configure(dev);
448                 if (ret != VMXNET3_SUCCESS)
449                         return ret;
450
451                 devRead->misc.uptFeatures |= VMXNET3_F_RSS;
452                 devRead->rssConfDesc.confVer = 1;
453                 devRead->rssConfDesc.confLen = sizeof(struct VMXNET3_RSSConf);
454                 devRead->rssConfDesc.confPA  = hw->rss_confPA;
455         }
456
457         if (dev->data->dev_conf.rxmode.hw_vlan_filter) {
458                 ret = vmxnet3_vlan_configure(dev);
459                 if (ret != VMXNET3_SUCCESS)
460                         return ret;
461         }
462
463         PMD_INIT_LOG(DEBUG,
464                      "Writing MAC Address : %02x:%02x:%02x:%02x:%02x:%02x",
465                      hw->perm_addr[0], hw->perm_addr[1], hw->perm_addr[2],
466                      hw->perm_addr[3], hw->perm_addr[4], hw->perm_addr[5]);
467
468         /* Write MAC Address back to device */
469         mac_ptr = (uint32_t *)hw->perm_addr;
470         val = *mac_ptr;
471         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_MACL, val);
472
473         val = (hw->perm_addr[5] << 8) | hw->perm_addr[4];
474         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_MACH, val);
475
476         return VMXNET3_SUCCESS;
477 }
478
479 /*
480  * Configure device link speed and setup link.
481  * Must be called after eth_vmxnet3_dev_init. Other wise it might fail
482  * It returns 0 on success.
483  */
484 static int
485 vmxnet3_dev_start(struct rte_eth_dev *dev)
486 {
487         int status, ret;
488         struct vmxnet3_hw *hw = dev->data->dev_private;
489
490         PMD_INIT_FUNC_TRACE();
491
492         ret = vmxnet3_setup_driver_shared(dev);
493         if (ret != VMXNET3_SUCCESS)
494                 return ret;
495
496         /* Exchange shared data with device */
497         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL,
498                                VMXNET3_GET_ADDR_LO(hw->sharedPA));
499         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAH,
500                                VMXNET3_GET_ADDR_HI(hw->sharedPA));
501
502         /* Activate device by register write */
503         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_ACTIVATE_DEV);
504         status = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD);
505
506         if (status != 0) {
507                 PMD_INIT_LOG(ERR, "Device activation in %s(): UNSUCCESSFUL", __func__);
508                 return -1;
509         }
510
511         /* Disable interrupts */
512         vmxnet3_disable_intr(hw);
513
514         /*
515          * Load RX queues with blank mbufs and update next2fill index for device
516          * Update RxMode of the device
517          */
518         ret = vmxnet3_dev_rxtx_init(dev);
519         if (ret != VMXNET3_SUCCESS) {
520                 PMD_INIT_LOG(ERR, "Device receive init in %s: UNSUCCESSFUL", __func__);
521                 return ret;
522         }
523
524         /* Setting proper Rx Mode and issue Rx Mode Update command */
525         vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_UCAST | VMXNET3_RXM_BCAST, 1);
526
527         /*
528          * Don't need to handle events for now
529          */
530 #if PROCESS_SYS_EVENTS == 1
531         events = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_ECR);
532         PMD_INIT_LOG(DEBUG, "Reading events: 0x%X", events);
533         vmxnet3_process_events(hw);
534 #endif
535         return status;
536 }
537
538 /*
539  * Stop device: disable rx and tx functions to allow for reconfiguring.
540  */
541 static void
542 vmxnet3_dev_stop(struct rte_eth_dev *dev)
543 {
544         struct rte_eth_link link;
545         struct vmxnet3_hw *hw = dev->data->dev_private;
546
547         PMD_INIT_FUNC_TRACE();
548
549         if (hw->adapter_stopped == TRUE) {
550                 PMD_INIT_LOG(DEBUG, "Device already closed.");
551                 return;
552         }
553
554         /* disable interrupts */
555         vmxnet3_disable_intr(hw);
556
557         /* quiesce the device first */
558         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
559         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL, 0);
560         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAH, 0);
561
562         /* reset the device */
563         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
564         PMD_INIT_LOG(DEBUG, "Device reset.");
565         hw->adapter_stopped = FALSE;
566
567         vmxnet3_dev_clear_queues(dev);
568
569         /* Clear recorded link status */
570         memset(&link, 0, sizeof(link));
571         rte_vmxnet3_dev_atomic_write_link_status(dev, &link);
572 }
573
574 /*
575  * Reset and stop device.
576  */
577 static void
578 vmxnet3_dev_close(struct rte_eth_dev *dev)
579 {
580         struct vmxnet3_hw *hw = dev->data->dev_private;
581
582         PMD_INIT_FUNC_TRACE();
583
584         vmxnet3_dev_stop(dev);
585         hw->adapter_stopped = TRUE;
586
587 }
588
589 static void
590 vmxnet3_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
591 {
592         unsigned int i;
593         struct vmxnet3_hw *hw = dev->data->dev_private;
594
595         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
596
597         RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_TX_QUEUES);
598         for (i = 0; i < hw->num_tx_queues; i++) {
599                 struct UPT1_TxStats *txStats = &hw->tqd_start[i].stats;
600
601                 stats->q_opackets[i] = txStats->ucastPktsTxOK +
602                         txStats->mcastPktsTxOK +
603                         txStats->bcastPktsTxOK;
604                 stats->q_obytes[i] = txStats->ucastBytesTxOK +
605                         txStats->mcastBytesTxOK +
606                         txStats->bcastBytesTxOK;
607
608                 stats->opackets += stats->q_opackets[i];
609                 stats->obytes += stats->q_obytes[i];
610                 stats->oerrors += txStats->pktsTxError +
611                         txStats->pktsTxDiscard;
612         }
613
614         RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_RX_QUEUES);
615         for (i = 0; i < hw->num_rx_queues; i++) {
616                 struct UPT1_RxStats *rxStats = &hw->rqd_start[i].stats;
617
618                 stats->q_ipackets[i] = rxStats->ucastPktsRxOK +
619                         rxStats->mcastPktsRxOK +
620                         rxStats->bcastPktsRxOK;
621
622                 stats->q_ibytes[i] = rxStats->ucastBytesRxOK +
623                         rxStats->mcastBytesRxOK +
624                         rxStats->bcastBytesRxOK;
625
626                 stats->ipackets += stats->q_ipackets[i];
627                 stats->ibytes += stats->q_ibytes[i];
628
629                 stats->q_errors[i] = rxStats->pktsRxError;
630                 stats->ierrors += rxStats->pktsRxError;
631                 stats->imcasts += rxStats->mcastPktsRxOK;
632                 stats->rx_nombuf += rxStats->pktsRxOutOfBuf;
633         }
634 }
635
636 static void
637 vmxnet3_dev_info_get(__attribute__((unused))struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
638 {
639         dev_info->max_rx_queues = VMXNET3_MAX_RX_QUEUES;
640         dev_info->max_tx_queues = VMXNET3_MAX_TX_QUEUES;
641         dev_info->min_rx_bufsize = 1518 + RTE_PKTMBUF_HEADROOM;
642         dev_info->max_rx_pktlen = 16384; /* includes CRC, cf MAXFRS register */
643         dev_info->max_mac_addrs = VMXNET3_MAX_MAC_ADDRS;
644 }
645
646 /* return 0 means link status changed, -1 means not changed */
647 static int
648 vmxnet3_dev_link_update(struct rte_eth_dev *dev, __attribute__((unused)) int wait_to_complete)
649 {
650         struct vmxnet3_hw *hw = dev->data->dev_private;
651         struct rte_eth_link link;
652         uint32_t ret;
653
654         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
655         ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD);
656
657         if (!ret) {
658                 PMD_INIT_LOG(ERR, "Link Status Negative : %s()", __func__);
659                 return -1;
660         }
661
662         if (ret & 0x1) {
663                 link.link_status = 1;
664                 link.link_duplex = ETH_LINK_FULL_DUPLEX;
665                 link.link_speed = ETH_LINK_SPEED_10000;
666
667                 rte_vmxnet3_dev_atomic_write_link_status(dev, &link);
668
669                 return 0;
670         }
671
672         return -1;
673 }
674
675 /* Updating rxmode through Vmxnet3_DriverShared structure in adapter */
676 static void
677 vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set) {
678
679         struct Vmxnet3_RxFilterConf *rxConf = &hw->shared->devRead.rxFilterConf;
680
681         if (set)
682                 rxConf->rxMode = rxConf->rxMode | feature;
683         else
684                 rxConf->rxMode = rxConf->rxMode & (~feature);
685
686         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_UPDATE_RX_MODE);
687 }
688
689 /* Promiscuous supported only if Vmxnet3_DriverShared is initialized in adapter */
690 static void
691 vmxnet3_dev_promiscuous_enable(struct rte_eth_dev *dev)
692 {
693         struct vmxnet3_hw *hw = dev->data->dev_private;
694
695         vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_PROMISC, 1);
696 }
697
698 /* Promiscuous supported only if Vmxnet3_DriverShared is initialized in adapter */
699 static void
700 vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev)
701 {
702         struct vmxnet3_hw *hw = dev->data->dev_private;
703
704         vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_PROMISC, 0);
705 }
706
707 /* Allmulticast supported only if Vmxnet3_DriverShared is initialized in adapter */
708 static void
709 vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev)
710 {
711         struct vmxnet3_hw *hw = dev->data->dev_private;
712
713         vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_ALL_MULTI, 1);
714 }
715
716 /* Allmulticast supported only if Vmxnet3_DriverShared is initialized in adapter */
717 static void
718 vmxnet3_dev_allmulticast_disable(struct rte_eth_dev *dev)
719 {
720         struct vmxnet3_hw *hw = dev->data->dev_private;
721
722         vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_ALL_MULTI, 0);
723 }
724
725 #if PROCESS_SYS_EVENTS == 1
726 static void
727 vmxnet3_process_events(struct vmxnet3_hw *hw)
728 {
729         uint32_t events = hw->shared->ecr;
730
731         if (!events) {
732                 PMD_INIT_LOG(ERR, "No events to process in %s()", __func__);
733                 return;
734         }
735
736         /*
737          * ECR bits when written with 1b are cleared. Hence write
738          * events back to ECR so that the bits which were set will be reset.
739          */
740         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_ECR, events);
741
742         /* Check if link state has changed */
743         if (events & VMXNET3_ECR_LINK)
744                 PMD_INIT_LOG(ERR,
745                              "Process events in %s(): VMXNET3_ECR_LINK event", __func__);
746
747         /* Check if there is an error on xmit/recv queues */
748         if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
749                 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_QUEUE_STATUS);
750
751                 if (hw->tqd_start->status.stopped)
752                         PMD_INIT_LOG(ERR, "tq error 0x%x",
753                                      hw->tqd_start->status.error);
754
755                 if (hw->rqd_start->status.stopped)
756                         PMD_INIT_LOG(ERR, "rq error 0x%x",
757                                      hw->rqd_start->status.error);
758
759                 /* Reset the device */
760                 /* Have to reset the device */
761         }
762
763         if (events & VMXNET3_ECR_DIC)
764                 PMD_INIT_LOG(ERR, "Device implementation change event.");
765
766         if (events & VMXNET3_ECR_DEBUG)
767                 PMD_INIT_LOG(ERR, "Debug event generated by device.");
768
769 }
770 #endif
771
772 static struct rte_driver rte_vmxnet3_driver = {
773         .type = PMD_PDEV,
774         .init = rte_vmxnet3_pmd_init,
775 };
776
777 PMD_REGISTER_DRIVER(rte_vmxnet3_driver);