vdpa/mlx5: support queue update
[dpdk.git] / drivers / net / virtio / virtio_user_ethdev.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2016 Intel Corporation
3  */
4
5 #include <stdint.h>
6 #include <sys/types.h>
7 #include <unistd.h>
8 #include <fcntl.h>
9 #include <sys/socket.h>
10
11 #include <rte_malloc.h>
12 #include <rte_kvargs.h>
13 #include <rte_ethdev_vdev.h>
14 #include <rte_bus_vdev.h>
15 #include <rte_alarm.h>
16 #include <rte_cycles.h>
17
18 #include "virtio_ethdev.h"
19 #include "virtio_logs.h"
20 #include "virtio_pci.h"
21 #include "virtqueue.h"
22 #include "virtio_rxtx.h"
23 #include "virtio_user/virtio_user_dev.h"
24 #include "virtio_user/vhost.h"
25
26 #define virtio_user_get_dev(hw) \
27         ((struct virtio_user_dev *)(hw)->virtio_user_dev)
28
29 static void
30 virtio_user_reset_queues_packed(struct rte_eth_dev *dev)
31 {
32         struct virtio_hw *hw = dev->data->dev_private;
33         struct virtnet_rx *rxvq;
34         struct virtnet_tx *txvq;
35         uint16_t i;
36
37         /* Add lock to avoid queue contention. */
38         rte_spinlock_lock(&hw->state_lock);
39         hw->started = 0;
40
41         /*
42          * Waitting for datapath to complete before resetting queues.
43          * 1 ms should be enough for the ongoing Tx/Rx function to finish.
44          */
45         rte_delay_ms(1);
46
47         /* Vring reset for each Tx queue and Rx queue. */
48         for (i = 0; i < dev->data->nb_rx_queues; i++) {
49                 rxvq = dev->data->rx_queues[i];
50                 virtqueue_rxvq_reset_packed(rxvq->vq);
51                 virtio_dev_rx_queue_setup_finish(dev, i);
52         }
53
54         for (i = 0; i < dev->data->nb_tx_queues; i++) {
55                 txvq = dev->data->tx_queues[i];
56                 virtqueue_txvq_reset_packed(txvq->vq);
57         }
58
59         hw->started = 1;
60         rte_spinlock_unlock(&hw->state_lock);
61 }
62
63
64 static int
65 virtio_user_server_reconnect(struct virtio_user_dev *dev)
66 {
67         int ret;
68         int connectfd;
69         struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id];
70         struct virtio_hw *hw = eth_dev->data->dev_private;
71
72         connectfd = accept(dev->listenfd, NULL, NULL);
73         if (connectfd < 0)
74                 return -1;
75
76         dev->vhostfd = connectfd;
77         if (dev->ops->send_request(dev, VHOST_USER_GET_FEATURES,
78                                    &dev->device_features) < 0) {
79                 PMD_INIT_LOG(ERR, "get_features failed: %s",
80                              strerror(errno));
81                 return -1;
82         }
83
84         dev->device_features |= dev->frontend_features;
85
86         /* umask vhost-user unsupported features */
87         dev->device_features &= ~(dev->unsupported_features);
88
89         dev->features &= dev->device_features;
90
91         /* For packed ring, resetting queues is required in reconnection. */
92         if (vtpci_packed_queue(hw) &&
93            (vtpci_get_status(hw) & VIRTIO_CONFIG_STATUS_DRIVER_OK)) {
94                 PMD_INIT_LOG(NOTICE, "Packets on the fly will be dropped"
95                                 " when packed ring reconnecting.");
96                 virtio_user_reset_queues_packed(eth_dev);
97         }
98
99         ret = virtio_user_start_device(dev);
100         if (ret < 0)
101                 return -1;
102
103         if (dev->queue_pairs > 1) {
104                 ret = virtio_user_handle_mq(dev, dev->queue_pairs);
105                 if (ret != 0) {
106                         PMD_INIT_LOG(ERR, "Fails to enable multi-queue pairs!");
107                         return -1;
108                 }
109         }
110         if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
111                 if (rte_intr_disable(eth_dev->intr_handle) < 0) {
112                         PMD_DRV_LOG(ERR, "interrupt disable failed");
113                         return -1;
114                 }
115                 rte_intr_callback_unregister(eth_dev->intr_handle,
116                                              virtio_interrupt_handler,
117                                              eth_dev);
118                 eth_dev->intr_handle->fd = connectfd;
119                 rte_intr_callback_register(eth_dev->intr_handle,
120                                            virtio_interrupt_handler, eth_dev);
121
122                 if (rte_intr_enable(eth_dev->intr_handle) < 0) {
123                         PMD_DRV_LOG(ERR, "interrupt enable failed");
124                         return -1;
125                 }
126         }
127         PMD_INIT_LOG(NOTICE, "server mode virtio-user reconnection succeeds!");
128         return 0;
129 }
130
131 static void
132 virtio_user_delayed_handler(void *param)
133 {
134         struct virtio_hw *hw = (struct virtio_hw *)param;
135         struct rte_eth_dev *eth_dev = &rte_eth_devices[hw->port_id];
136         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
137
138         if (rte_intr_disable(eth_dev->intr_handle) < 0) {
139                 PMD_DRV_LOG(ERR, "interrupt disable failed");
140                 return;
141         }
142         rte_intr_callback_unregister(eth_dev->intr_handle,
143                                      virtio_interrupt_handler, eth_dev);
144         if (dev->is_server) {
145                 if (dev->vhostfd >= 0) {
146                         close(dev->vhostfd);
147                         dev->vhostfd = -1;
148                 }
149                 eth_dev->intr_handle->fd = dev->listenfd;
150                 rte_intr_callback_register(eth_dev->intr_handle,
151                                            virtio_interrupt_handler, eth_dev);
152                 if (rte_intr_enable(eth_dev->intr_handle) < 0) {
153                         PMD_DRV_LOG(ERR, "interrupt enable failed");
154                         return;
155                 }
156         }
157 }
158
159 static void
160 virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset,
161                      void *dst, int length)
162 {
163         int i;
164         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
165
166         if (offset == offsetof(struct virtio_net_config, mac) &&
167             length == RTE_ETHER_ADDR_LEN) {
168                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i)
169                         ((uint8_t *)dst)[i] = dev->mac_addr[i];
170                 return;
171         }
172
173         if (offset == offsetof(struct virtio_net_config, status)) {
174                 char buf[128];
175
176                 if (dev->vhostfd >= 0) {
177                         int r;
178                         int flags;
179
180                         flags = fcntl(dev->vhostfd, F_GETFL);
181                         if (fcntl(dev->vhostfd, F_SETFL,
182                                         flags | O_NONBLOCK) == -1) {
183                                 PMD_DRV_LOG(ERR, "error setting O_NONBLOCK flag");
184                                 return;
185                         }
186                         r = recv(dev->vhostfd, buf, 128, MSG_PEEK);
187                         if (r == 0 || (r < 0 && errno != EAGAIN)) {
188                                 dev->status &= (~VIRTIO_NET_S_LINK_UP);
189                                 PMD_DRV_LOG(ERR, "virtio-user port %u is down",
190                                             hw->port_id);
191
192                                 /* This function could be called in the process
193                                  * of interrupt handling, callback cannot be
194                                  * unregistered here, set an alarm to do it.
195                                  */
196                                 rte_eal_alarm_set(1,
197                                                   virtio_user_delayed_handler,
198                                                   (void *)hw);
199                         } else {
200                                 dev->status |= VIRTIO_NET_S_LINK_UP;
201                         }
202                         if (fcntl(dev->vhostfd, F_SETFL,
203                                         flags & ~O_NONBLOCK) == -1) {
204                                 PMD_DRV_LOG(ERR, "error clearing O_NONBLOCK flag");
205                                 return;
206                         }
207                 } else if (dev->is_server) {
208                         dev->status &= (~VIRTIO_NET_S_LINK_UP);
209                         if (virtio_user_server_reconnect(dev) >= 0)
210                                 dev->status |= VIRTIO_NET_S_LINK_UP;
211                 }
212
213                 *(uint16_t *)dst = dev->status;
214         }
215
216         if (offset == offsetof(struct virtio_net_config, max_virtqueue_pairs))
217                 *(uint16_t *)dst = dev->max_queue_pairs;
218 }
219
220 static void
221 virtio_user_write_dev_config(struct virtio_hw *hw, size_t offset,
222                       const void *src, int length)
223 {
224         int i;
225         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
226
227         if ((offset == offsetof(struct virtio_net_config, mac)) &&
228             (length == RTE_ETHER_ADDR_LEN))
229                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i)
230                         dev->mac_addr[i] = ((const uint8_t *)src)[i];
231         else
232                 PMD_DRV_LOG(ERR, "not supported offset=%zu, len=%d",
233                             offset, length);
234 }
235
236 static void
237 virtio_user_reset(struct virtio_hw *hw)
238 {
239         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
240
241         if (dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK)
242                 virtio_user_stop_device(dev);
243 }
244
245 static void
246 virtio_user_set_status(struct virtio_hw *hw, uint8_t status)
247 {
248         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
249
250         if (status & VIRTIO_CONFIG_STATUS_DRIVER_OK)
251                 virtio_user_start_device(dev);
252         else if (status == VIRTIO_CONFIG_STATUS_RESET)
253                 virtio_user_reset(hw);
254         dev->status = status;
255 }
256
257 static uint8_t
258 virtio_user_get_status(struct virtio_hw *hw)
259 {
260         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
261
262         return dev->status;
263 }
264
265 static uint64_t
266 virtio_user_get_features(struct virtio_hw *hw)
267 {
268         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
269
270         /* unmask feature bits defined in vhost user protocol */
271         return dev->device_features & VIRTIO_PMD_SUPPORTED_GUEST_FEATURES;
272 }
273
274 static void
275 virtio_user_set_features(struct virtio_hw *hw, uint64_t features)
276 {
277         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
278
279         dev->features = features & dev->device_features;
280 }
281
282 static uint8_t
283 virtio_user_get_isr(struct virtio_hw *hw __rte_unused)
284 {
285         /* rxq interrupts and config interrupt are separated in virtio-user,
286          * here we only report config change.
287          */
288         return VIRTIO_PCI_ISR_CONFIG;
289 }
290
291 static uint16_t
292 virtio_user_set_config_irq(struct virtio_hw *hw __rte_unused,
293                     uint16_t vec __rte_unused)
294 {
295         return 0;
296 }
297
298 static uint16_t
299 virtio_user_set_queue_irq(struct virtio_hw *hw __rte_unused,
300                           struct virtqueue *vq __rte_unused,
301                           uint16_t vec)
302 {
303         /* pretend we have done that */
304         return vec;
305 }
306
307 /* This function is to get the queue size, aka, number of descs, of a specified
308  * queue. Different with the VHOST_USER_GET_QUEUE_NUM, which is used to get the
309  * max supported queues.
310  */
311 static uint16_t
312 virtio_user_get_queue_num(struct virtio_hw *hw, uint16_t queue_id __rte_unused)
313 {
314         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
315
316         /* Currently, each queue has same queue size */
317         return dev->queue_size;
318 }
319
320 static void
321 virtio_user_setup_queue_packed(struct virtqueue *vq,
322                                struct virtio_user_dev *dev)
323 {
324         uint16_t queue_idx = vq->vq_queue_index;
325         struct vring_packed *vring;
326         uint64_t desc_addr;
327         uint64_t avail_addr;
328         uint64_t used_addr;
329         uint16_t i;
330
331         vring  = &dev->packed_vrings[queue_idx];
332         desc_addr = (uintptr_t)vq->vq_ring_virt_mem;
333         avail_addr = desc_addr + vq->vq_nentries *
334                 sizeof(struct vring_packed_desc);
335         used_addr = RTE_ALIGN_CEIL(avail_addr +
336                            sizeof(struct vring_packed_desc_event),
337                            VIRTIO_PCI_VRING_ALIGN);
338         vring->num = vq->vq_nentries;
339         vring->desc = (void *)(uintptr_t)desc_addr;
340         vring->driver = (void *)(uintptr_t)avail_addr;
341         vring->device = (void *)(uintptr_t)used_addr;
342         dev->packed_queues[queue_idx].avail_wrap_counter = true;
343         dev->packed_queues[queue_idx].used_wrap_counter = true;
344
345         for (i = 0; i < vring->num; i++)
346                 vring->desc[i].flags = 0;
347 }
348
349 static void
350 virtio_user_setup_queue_split(struct virtqueue *vq, struct virtio_user_dev *dev)
351 {
352         uint16_t queue_idx = vq->vq_queue_index;
353         uint64_t desc_addr, avail_addr, used_addr;
354
355         desc_addr = (uintptr_t)vq->vq_ring_virt_mem;
356         avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc);
357         used_addr = RTE_ALIGN_CEIL(avail_addr + offsetof(struct vring_avail,
358                                                          ring[vq->vq_nentries]),
359                                    VIRTIO_PCI_VRING_ALIGN);
360
361         dev->vrings[queue_idx].num = vq->vq_nentries;
362         dev->vrings[queue_idx].desc = (void *)(uintptr_t)desc_addr;
363         dev->vrings[queue_idx].avail = (void *)(uintptr_t)avail_addr;
364         dev->vrings[queue_idx].used = (void *)(uintptr_t)used_addr;
365 }
366
367 static int
368 virtio_user_setup_queue(struct virtio_hw *hw, struct virtqueue *vq)
369 {
370         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
371
372         if (vtpci_packed_queue(hw))
373                 virtio_user_setup_queue_packed(vq, dev);
374         else
375                 virtio_user_setup_queue_split(vq, dev);
376
377         return 0;
378 }
379
380 static void
381 virtio_user_del_queue(struct virtio_hw *hw, struct virtqueue *vq)
382 {
383         /* For legacy devices, write 0 to VIRTIO_PCI_QUEUE_PFN port, QEMU
384          * correspondingly stops the ioeventfds, and reset the status of
385          * the device.
386          * For modern devices, set queue desc, avail, used in PCI bar to 0,
387          * not see any more behavior in QEMU.
388          *
389          * Here we just care about what information to deliver to vhost-user
390          * or vhost-kernel. So we just close ioeventfd for now.
391          */
392         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
393
394         close(dev->callfds[vq->vq_queue_index]);
395         close(dev->kickfds[vq->vq_queue_index]);
396 }
397
398 static void
399 virtio_user_notify_queue(struct virtio_hw *hw, struct virtqueue *vq)
400 {
401         uint64_t buf = 1;
402         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
403
404         if (hw->cvq && (hw->cvq->vq == vq)) {
405                 if (vtpci_packed_queue(vq->hw))
406                         virtio_user_handle_cq_packed(dev, vq->vq_queue_index);
407                 else
408                         virtio_user_handle_cq(dev, vq->vq_queue_index);
409                 return;
410         }
411
412         if (write(dev->kickfds[vq->vq_queue_index], &buf, sizeof(buf)) < 0)
413                 PMD_DRV_LOG(ERR, "failed to kick backend: %s",
414                             strerror(errno));
415 }
416
417 const struct virtio_pci_ops virtio_user_ops = {
418         .read_dev_cfg   = virtio_user_read_dev_config,
419         .write_dev_cfg  = virtio_user_write_dev_config,
420         .get_status     = virtio_user_get_status,
421         .set_status     = virtio_user_set_status,
422         .get_features   = virtio_user_get_features,
423         .set_features   = virtio_user_set_features,
424         .get_isr        = virtio_user_get_isr,
425         .set_config_irq = virtio_user_set_config_irq,
426         .set_queue_irq  = virtio_user_set_queue_irq,
427         .get_queue_num  = virtio_user_get_queue_num,
428         .setup_queue    = virtio_user_setup_queue,
429         .del_queue      = virtio_user_del_queue,
430         .notify_queue   = virtio_user_notify_queue,
431 };
432
433 static const char *valid_args[] = {
434 #define VIRTIO_USER_ARG_QUEUES_NUM     "queues"
435         VIRTIO_USER_ARG_QUEUES_NUM,
436 #define VIRTIO_USER_ARG_CQ_NUM         "cq"
437         VIRTIO_USER_ARG_CQ_NUM,
438 #define VIRTIO_USER_ARG_MAC            "mac"
439         VIRTIO_USER_ARG_MAC,
440 #define VIRTIO_USER_ARG_PATH           "path"
441         VIRTIO_USER_ARG_PATH,
442 #define VIRTIO_USER_ARG_QUEUE_SIZE     "queue_size"
443         VIRTIO_USER_ARG_QUEUE_SIZE,
444 #define VIRTIO_USER_ARG_INTERFACE_NAME "iface"
445         VIRTIO_USER_ARG_INTERFACE_NAME,
446 #define VIRTIO_USER_ARG_SERVER_MODE    "server"
447         VIRTIO_USER_ARG_SERVER_MODE,
448 #define VIRTIO_USER_ARG_MRG_RXBUF      "mrg_rxbuf"
449         VIRTIO_USER_ARG_MRG_RXBUF,
450 #define VIRTIO_USER_ARG_IN_ORDER       "in_order"
451         VIRTIO_USER_ARG_IN_ORDER,
452 #define VIRTIO_USER_ARG_PACKED_VQ      "packed_vq"
453         VIRTIO_USER_ARG_PACKED_VQ,
454 #define VIRTIO_USER_ARG_SPEED          "speed"
455         VIRTIO_USER_ARG_SPEED,
456 #define VIRTIO_USER_ARG_VECTORIZED     "vectorized"
457         VIRTIO_USER_ARG_VECTORIZED,
458         NULL
459 };
460
461 #define VIRTIO_USER_DEF_CQ_EN   0
462 #define VIRTIO_USER_DEF_Q_NUM   1
463 #define VIRTIO_USER_DEF_Q_SZ    256
464 #define VIRTIO_USER_DEF_SERVER_MODE     0
465
466 static int
467 get_string_arg(const char *key __rte_unused,
468                const char *value, void *extra_args)
469 {
470         if (!value || !extra_args)
471                 return -EINVAL;
472
473         *(char **)extra_args = strdup(value);
474
475         if (!*(char **)extra_args)
476                 return -ENOMEM;
477
478         return 0;
479 }
480
481 static int
482 get_integer_arg(const char *key __rte_unused,
483                 const char *value, void *extra_args)
484 {
485         uint64_t integer = 0;
486         if (!value || !extra_args)
487                 return -EINVAL;
488         errno = 0;
489         integer = strtoull(value, NULL, 0);
490         /* extra_args keeps default value, it should be replaced
491          * only in case of successful parsing of the 'value' arg
492          */
493         if (errno == 0)
494                 *(uint64_t *)extra_args = integer;
495         return -errno;
496 }
497
498 static struct rte_eth_dev *
499 virtio_user_eth_dev_alloc(struct rte_vdev_device *vdev)
500 {
501         struct rte_eth_dev *eth_dev;
502         struct rte_eth_dev_data *data;
503         struct virtio_hw *hw;
504         struct virtio_user_dev *dev;
505
506         eth_dev = rte_eth_vdev_allocate(vdev, sizeof(*hw));
507         if (!eth_dev) {
508                 PMD_INIT_LOG(ERR, "cannot alloc rte_eth_dev");
509                 return NULL;
510         }
511
512         data = eth_dev->data;
513         hw = eth_dev->data->dev_private;
514
515         dev = rte_zmalloc(NULL, sizeof(*dev), 0);
516         if (!dev) {
517                 PMD_INIT_LOG(ERR, "malloc virtio_user_dev failed");
518                 rte_eth_dev_release_port(eth_dev);
519                 return NULL;
520         }
521
522         hw->port_id = data->port_id;
523         dev->port_id = data->port_id;
524         virtio_hw_internal[hw->port_id].vtpci_ops = &virtio_user_ops;
525         /*
526          * MSIX is required to enable LSC (see virtio_init_device).
527          * Here just pretend that we support msix.
528          */
529         hw->use_msix = 1;
530         hw->modern   = 0;
531         hw->use_vec_rx = 0;
532         hw->use_vec_tx = 0;
533         hw->use_inorder_rx = 0;
534         hw->use_inorder_tx = 0;
535         hw->virtio_user_dev = dev;
536         return eth_dev;
537 }
538
539 static void
540 virtio_user_eth_dev_free(struct rte_eth_dev *eth_dev)
541 {
542         struct rte_eth_dev_data *data = eth_dev->data;
543         struct virtio_hw *hw = data->dev_private;
544
545         rte_free(hw->virtio_user_dev);
546         rte_eth_dev_release_port(eth_dev);
547 }
548
549 /* Dev initialization routine. Invoked once for each virtio vdev at
550  * EAL init time, see rte_bus_probe().
551  * Returns 0 on success.
552  */
553 static int
554 virtio_user_pmd_probe(struct rte_vdev_device *dev)
555 {
556         struct rte_kvargs *kvlist = NULL;
557         struct rte_eth_dev *eth_dev;
558         struct virtio_hw *hw;
559         uint64_t queues = VIRTIO_USER_DEF_Q_NUM;
560         uint64_t cq = VIRTIO_USER_DEF_CQ_EN;
561         uint64_t queue_size = VIRTIO_USER_DEF_Q_SZ;
562         uint64_t server_mode = VIRTIO_USER_DEF_SERVER_MODE;
563         uint64_t mrg_rxbuf = 1;
564         uint64_t in_order = 1;
565         uint64_t packed_vq = 0;
566         uint64_t vectorized = 0;
567         char *path = NULL;
568         char *ifname = NULL;
569         char *mac_addr = NULL;
570         int ret = -1;
571
572         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
573                 const char *name = rte_vdev_device_name(dev);
574                 eth_dev = rte_eth_dev_attach_secondary(name);
575                 if (!eth_dev) {
576                         PMD_INIT_LOG(ERR, "Failed to probe %s", name);
577                         return -1;
578                 }
579
580                 if (eth_virtio_dev_init(eth_dev) < 0) {
581                         PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails");
582                         rte_eth_dev_release_port(eth_dev);
583                         return -1;
584                 }
585
586                 eth_dev->dev_ops = &virtio_user_secondary_eth_dev_ops;
587                 eth_dev->device = &dev->device;
588                 rte_eth_dev_probing_finish(eth_dev);
589                 return 0;
590         }
591
592         kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_args);
593         if (!kvlist) {
594                 PMD_INIT_LOG(ERR, "error when parsing param");
595                 goto end;
596         }
597
598         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_PATH) == 1) {
599                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_PATH,
600                                        &get_string_arg, &path) < 0) {
601                         PMD_INIT_LOG(ERR, "error to parse %s",
602                                      VIRTIO_USER_ARG_PATH);
603                         goto end;
604                 }
605         } else {
606                 PMD_INIT_LOG(ERR, "arg %s is mandatory for virtio_user",
607                              VIRTIO_USER_ARG_PATH);
608                 goto end;
609         }
610
611         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_INTERFACE_NAME) == 1) {
612                 if (is_vhost_user_by_type(path)) {
613                         PMD_INIT_LOG(ERR,
614                                 "arg %s applies only to vhost-kernel backend",
615                                 VIRTIO_USER_ARG_INTERFACE_NAME);
616                         goto end;
617                 }
618
619                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_INTERFACE_NAME,
620                                        &get_string_arg, &ifname) < 0) {
621                         PMD_INIT_LOG(ERR, "error to parse %s",
622                                      VIRTIO_USER_ARG_INTERFACE_NAME);
623                         goto end;
624                 }
625         }
626
627         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_MAC) == 1) {
628                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MAC,
629                                        &get_string_arg, &mac_addr) < 0) {
630                         PMD_INIT_LOG(ERR, "error to parse %s",
631                                      VIRTIO_USER_ARG_MAC);
632                         goto end;
633                 }
634         }
635
636         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE) == 1) {
637                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE,
638                                        &get_integer_arg, &queue_size) < 0) {
639                         PMD_INIT_LOG(ERR, "error to parse %s",
640                                      VIRTIO_USER_ARG_QUEUE_SIZE);
641                         goto end;
642                 }
643         }
644
645         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_QUEUES_NUM) == 1) {
646                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUES_NUM,
647                                        &get_integer_arg, &queues) < 0) {
648                         PMD_INIT_LOG(ERR, "error to parse %s",
649                                      VIRTIO_USER_ARG_QUEUES_NUM);
650                         goto end;
651                 }
652         }
653
654         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_SERVER_MODE) == 1) {
655                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_SERVER_MODE,
656                                        &get_integer_arg, &server_mode) < 0) {
657                         PMD_INIT_LOG(ERR, "error to parse %s",
658                                      VIRTIO_USER_ARG_SERVER_MODE);
659                         goto end;
660                 }
661         }
662
663         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_CQ_NUM) == 1) {
664                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_CQ_NUM,
665                                        &get_integer_arg, &cq) < 0) {
666                         PMD_INIT_LOG(ERR, "error to parse %s",
667                                      VIRTIO_USER_ARG_CQ_NUM);
668                         goto end;
669                 }
670         } else if (queues > 1) {
671                 cq = 1;
672         }
673
674         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_PACKED_VQ) == 1) {
675                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_PACKED_VQ,
676                                        &get_integer_arg, &packed_vq) < 0) {
677                         PMD_INIT_LOG(ERR, "error to parse %s",
678                                      VIRTIO_USER_ARG_PACKED_VQ);
679                         goto end;
680                 }
681         }
682
683         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_VECTORIZED) == 1) {
684                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_VECTORIZED,
685                                        &get_integer_arg, &vectorized) < 0) {
686                         PMD_INIT_LOG(ERR, "error to parse %s",
687                                      VIRTIO_USER_ARG_VECTORIZED);
688                         goto end;
689                 }
690         }
691
692         if (queues > 1 && cq == 0) {
693                 PMD_INIT_LOG(ERR, "multi-q requires ctrl-q");
694                 goto end;
695         }
696
697         if (queues > VIRTIO_MAX_VIRTQUEUE_PAIRS) {
698                 PMD_INIT_LOG(ERR, "arg %s %" PRIu64 " exceeds the limit %u",
699                         VIRTIO_USER_ARG_QUEUES_NUM, queues,
700                         VIRTIO_MAX_VIRTQUEUE_PAIRS);
701                 goto end;
702         }
703
704         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_MRG_RXBUF) == 1) {
705                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MRG_RXBUF,
706                                        &get_integer_arg, &mrg_rxbuf) < 0) {
707                         PMD_INIT_LOG(ERR, "error to parse %s",
708                                      VIRTIO_USER_ARG_MRG_RXBUF);
709                         goto end;
710                 }
711         }
712
713         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_IN_ORDER) == 1) {
714                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_IN_ORDER,
715                                        &get_integer_arg, &in_order) < 0) {
716                         PMD_INIT_LOG(ERR, "error to parse %s",
717                                      VIRTIO_USER_ARG_IN_ORDER);
718                         goto end;
719                 }
720         }
721
722         eth_dev = virtio_user_eth_dev_alloc(dev);
723         if (!eth_dev) {
724                 PMD_INIT_LOG(ERR, "virtio_user fails to alloc device");
725                 goto end;
726         }
727
728         hw = eth_dev->data->dev_private;
729         if (virtio_user_dev_init(hw->virtio_user_dev, path, queues, cq,
730                          queue_size, mac_addr, &ifname, server_mode,
731                          mrg_rxbuf, in_order, packed_vq) < 0) {
732                 PMD_INIT_LOG(ERR, "virtio_user_dev_init fails");
733                 virtio_user_eth_dev_free(eth_dev);
734                 goto end;
735         }
736
737         /* previously called by pci probing for physical dev */
738         if (eth_virtio_dev_init(eth_dev) < 0) {
739                 PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails");
740                 virtio_user_eth_dev_free(eth_dev);
741                 goto end;
742         }
743
744         if (vectorized) {
745                 if (packed_vq) {
746 #if defined(CC_AVX512_SUPPORT)
747                         hw->use_vec_rx = 1;
748                         hw->use_vec_tx = 1;
749 #else
750                         PMD_INIT_LOG(INFO,
751                                 "building environment do not support packed ring vectorized");
752 #endif
753                 } else {
754                         hw->use_vec_rx = 1;
755                 }
756         }
757
758         rte_eth_dev_probing_finish(eth_dev);
759         ret = 0;
760
761 end:
762         if (kvlist)
763                 rte_kvargs_free(kvlist);
764         if (path)
765                 free(path);
766         if (mac_addr)
767                 free(mac_addr);
768         if (ifname)
769                 free(ifname);
770         return ret;
771 }
772
773 static int
774 virtio_user_pmd_remove(struct rte_vdev_device *vdev)
775 {
776         const char *name;
777         struct rte_eth_dev *eth_dev;
778
779         if (!vdev)
780                 return -EINVAL;
781
782         name = rte_vdev_device_name(vdev);
783         PMD_DRV_LOG(INFO, "Un-Initializing %s", name);
784         eth_dev = rte_eth_dev_allocated(name);
785         /* Port has already been released by close. */
786         if (!eth_dev)
787                 return 0;
788
789         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
790                 return rte_eth_dev_release_port(eth_dev);
791
792         /* make sure the device is stopped, queues freed */
793         rte_eth_dev_close(eth_dev->data->port_id);
794
795         return 0;
796 }
797
798 static struct rte_vdev_driver virtio_user_driver = {
799         .probe = virtio_user_pmd_probe,
800         .remove = virtio_user_pmd_remove,
801 };
802
803 RTE_PMD_REGISTER_VDEV(net_virtio_user, virtio_user_driver);
804 RTE_PMD_REGISTER_ALIAS(net_virtio_user, virtio_user);
805 RTE_PMD_REGISTER_PARAM_STRING(net_virtio_user,
806         "path=<path> "
807         "mac=<mac addr> "
808         "cq=<int> "
809         "queue_size=<int> "
810         "queues=<int> "
811         "iface=<string> "
812         "server=<0|1> "
813         "mrg_rxbuf=<0|1> "
814         "in_order=<0|1> "
815         "packed_vq=<0|1> "
816         "speed=<int> "
817         "vectorized=<0|1>");