net/virtio: introduce PCI device metadata
[dpdk.git] / drivers / net / virtio / virtio_user_ethdev.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2016 Intel Corporation
3  */
4
5 #include <stdint.h>
6 #include <sys/types.h>
7 #include <unistd.h>
8 #include <fcntl.h>
9 #include <linux/major.h>
10 #include <sys/stat.h>
11 #include <sys/sysmacros.h>
12 #include <sys/socket.h>
13
14 #include <rte_malloc.h>
15 #include <rte_kvargs.h>
16 #include <ethdev_vdev.h>
17 #include <rte_bus_vdev.h>
18 #include <rte_alarm.h>
19 #include <rte_cycles.h>
20
21 #include "virtio_ethdev.h"
22 #include "virtio_logs.h"
23 #include "virtio_pci.h"
24 #include "virtqueue.h"
25 #include "virtio_rxtx.h"
26 #include "virtio_user/virtio_user_dev.h"
27 #include "virtio_user/vhost.h"
28
29 #define virtio_user_get_dev(hwp) container_of(hwp, struct virtio_user_dev, hw)
30
31 static void
32 virtio_user_reset_queues_packed(struct rte_eth_dev *eth_dev)
33 {
34         struct virtio_user_dev *dev = eth_dev->data->dev_private;
35         struct virtio_hw *hw = &dev->hw;
36         struct virtnet_rx *rxvq;
37         struct virtnet_tx *txvq;
38         uint16_t i;
39
40         /* Add lock to avoid queue contention. */
41         rte_spinlock_lock(&hw->state_lock);
42         hw->started = 0;
43
44         /*
45          * Waitting for datapath to complete before resetting queues.
46          * 1 ms should be enough for the ongoing Tx/Rx function to finish.
47          */
48         rte_delay_ms(1);
49
50         /* Vring reset for each Tx queue and Rx queue. */
51         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
52                 rxvq = eth_dev->data->rx_queues[i];
53                 virtqueue_rxvq_reset_packed(rxvq->vq);
54                 virtio_dev_rx_queue_setup_finish(eth_dev, i);
55         }
56
57         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
58                 txvq = eth_dev->data->tx_queues[i];
59                 virtqueue_txvq_reset_packed(txvq->vq);
60         }
61
62         hw->started = 1;
63         rte_spinlock_unlock(&hw->state_lock);
64 }
65
66
67 static int
68 virtio_user_server_reconnect(struct virtio_user_dev *dev)
69 {
70         int ret, connectfd, old_status;
71         struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id];
72         struct virtio_hw *hw = &dev->hw;
73         uint64_t protocol_features;
74
75         connectfd = accept(dev->listenfd, NULL, NULL);
76         if (connectfd < 0)
77                 return -1;
78
79         dev->vhostfd = connectfd;
80         old_status = dev->status;
81
82         vtpci_reset(hw);
83
84         vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
85
86         vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
87
88         if (dev->ops->send_request(dev, VHOST_USER_GET_FEATURES,
89                                    &dev->device_features) < 0) {
90                 PMD_INIT_LOG(ERR, "get_features failed: %s",
91                              strerror(errno));
92                 return -1;
93         }
94
95         if (dev->device_features &
96                         (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)) {
97                 if (dev->ops->send_request(dev,
98                                         VHOST_USER_GET_PROTOCOL_FEATURES,
99                                         &protocol_features))
100                         return -1;
101
102                 /* Offer VHOST_USER_PROTOCOL_F_STATUS */
103                 dev->protocol_features |=
104                         (1ULL << VHOST_USER_PROTOCOL_F_STATUS);
105                 dev->protocol_features &= protocol_features;
106
107                 if (dev->ops->send_request(dev,
108                                         VHOST_USER_SET_PROTOCOL_FEATURES,
109                                         &dev->protocol_features))
110                         return -1;
111
112                 if (!(dev->protocol_features &
113                                 (1ULL << VHOST_USER_PROTOCOL_F_MQ)))
114                         dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ);
115         }
116
117         dev->device_features |= dev->frontend_features;
118
119         /* umask vhost-user unsupported features */
120         dev->device_features &= ~(dev->unsupported_features);
121
122         dev->features &= dev->device_features;
123
124         /* For packed ring, resetting queues is required in reconnection. */
125         if (vtpci_packed_queue(hw) &&
126            (old_status & VIRTIO_CONFIG_STATUS_DRIVER_OK)) {
127                 PMD_INIT_LOG(NOTICE, "Packets on the fly will be dropped"
128                                 " when packed ring reconnecting.");
129                 virtio_user_reset_queues_packed(eth_dev);
130         }
131
132         vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK);
133
134         /* Start the device */
135         vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER_OK);
136         if (!dev->started)
137                 return -1;
138
139         if (dev->queue_pairs > 1) {
140                 ret = virtio_user_handle_mq(dev, dev->queue_pairs);
141                 if (ret != 0) {
142                         PMD_INIT_LOG(ERR, "Fails to enable multi-queue pairs!");
143                         return -1;
144                 }
145         }
146         if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
147                 if (rte_intr_disable(eth_dev->intr_handle) < 0) {
148                         PMD_DRV_LOG(ERR, "interrupt disable failed");
149                         return -1;
150                 }
151                 rte_intr_callback_unregister(eth_dev->intr_handle,
152                                              virtio_interrupt_handler,
153                                              eth_dev);
154                 eth_dev->intr_handle->fd = connectfd;
155                 rte_intr_callback_register(eth_dev->intr_handle,
156                                            virtio_interrupt_handler, eth_dev);
157
158                 if (rte_intr_enable(eth_dev->intr_handle) < 0) {
159                         PMD_DRV_LOG(ERR, "interrupt enable failed");
160                         return -1;
161                 }
162         }
163         PMD_INIT_LOG(NOTICE, "server mode virtio-user reconnection succeeds!");
164         return 0;
165 }
166
167 static void
168 virtio_user_delayed_handler(void *param)
169 {
170         struct virtio_hw *hw = (struct virtio_hw *)param;
171         struct rte_eth_dev *eth_dev = &rte_eth_devices[hw->port_id];
172         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
173
174         if (rte_intr_disable(eth_dev->intr_handle) < 0) {
175                 PMD_DRV_LOG(ERR, "interrupt disable failed");
176                 return;
177         }
178         rte_intr_callback_unregister(eth_dev->intr_handle,
179                                      virtio_interrupt_handler, eth_dev);
180         if (dev->is_server) {
181                 if (dev->vhostfd >= 0) {
182                         close(dev->vhostfd);
183                         dev->vhostfd = -1;
184                         /* Until the featuers are negotiated again, don't assume
185                          * the backend supports VHOST_USER_PROTOCOL_F_STATUS
186                          */
187                         dev->protocol_features &=
188                                 ~(1ULL << VHOST_USER_PROTOCOL_F_STATUS);
189                 }
190                 eth_dev->intr_handle->fd = dev->listenfd;
191                 rte_intr_callback_register(eth_dev->intr_handle,
192                                            virtio_interrupt_handler, eth_dev);
193                 if (rte_intr_enable(eth_dev->intr_handle) < 0) {
194                         PMD_DRV_LOG(ERR, "interrupt enable failed");
195                         return;
196                 }
197         }
198 }
199
200 static void
201 virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset,
202                      void *dst, int length)
203 {
204         int i;
205         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
206
207         if (offset == offsetof(struct virtio_net_config, mac) &&
208             length == RTE_ETHER_ADDR_LEN) {
209                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i)
210                         ((uint8_t *)dst)[i] = dev->mac_addr[i];
211                 return;
212         }
213
214         if (offset == offsetof(struct virtio_net_config, status)) {
215                 char buf[128];
216
217                 if (dev->vhostfd >= 0) {
218                         int r;
219                         int flags;
220
221                         flags = fcntl(dev->vhostfd, F_GETFL);
222                         if (fcntl(dev->vhostfd, F_SETFL,
223                                         flags | O_NONBLOCK) == -1) {
224                                 PMD_DRV_LOG(ERR, "error setting O_NONBLOCK flag");
225                                 return;
226                         }
227                         r = recv(dev->vhostfd, buf, 128, MSG_PEEK);
228                         if (r == 0 || (r < 0 && errno != EAGAIN)) {
229                                 dev->net_status &= (~VIRTIO_NET_S_LINK_UP);
230                                 PMD_DRV_LOG(ERR, "virtio-user port %u is down",
231                                             hw->port_id);
232
233                                 /* This function could be called in the process
234                                  * of interrupt handling, callback cannot be
235                                  * unregistered here, set an alarm to do it.
236                                  */
237                                 rte_eal_alarm_set(1,
238                                                   virtio_user_delayed_handler,
239                                                   (void *)hw);
240                         } else {
241                                 dev->net_status |= VIRTIO_NET_S_LINK_UP;
242                         }
243                         if (fcntl(dev->vhostfd, F_SETFL,
244                                         flags & ~O_NONBLOCK) == -1) {
245                                 PMD_DRV_LOG(ERR, "error clearing O_NONBLOCK flag");
246                                 return;
247                         }
248                 } else if (dev->is_server) {
249                         dev->net_status &= (~VIRTIO_NET_S_LINK_UP);
250                         if (virtio_user_server_reconnect(dev) >= 0)
251                                 dev->net_status |= VIRTIO_NET_S_LINK_UP;
252                 }
253
254                 *(uint16_t *)dst = dev->net_status;
255         }
256
257         if (offset == offsetof(struct virtio_net_config, max_virtqueue_pairs))
258                 *(uint16_t *)dst = dev->max_queue_pairs;
259 }
260
261 static void
262 virtio_user_write_dev_config(struct virtio_hw *hw, size_t offset,
263                       const void *src, int length)
264 {
265         int i;
266         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
267
268         if ((offset == offsetof(struct virtio_net_config, mac)) &&
269             (length == RTE_ETHER_ADDR_LEN))
270                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i)
271                         dev->mac_addr[i] = ((const uint8_t *)src)[i];
272         else
273                 PMD_DRV_LOG(ERR, "not supported offset=%zu, len=%d",
274                             offset, length);
275 }
276
277 static void
278 virtio_user_reset(struct virtio_hw *hw)
279 {
280         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
281
282         if (dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK)
283                 virtio_user_stop_device(dev);
284 }
285
286 static void
287 virtio_user_set_status(struct virtio_hw *hw, uint8_t status)
288 {
289         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
290         uint8_t old_status = dev->status;
291
292         if (status & VIRTIO_CONFIG_STATUS_FEATURES_OK &&
293                         ~old_status & VIRTIO_CONFIG_STATUS_FEATURES_OK)
294                 virtio_user_dev_set_features(dev);
295         if (status & VIRTIO_CONFIG_STATUS_DRIVER_OK)
296                 virtio_user_start_device(dev);
297         else if (status == VIRTIO_CONFIG_STATUS_RESET)
298                 virtio_user_reset(hw);
299
300         virtio_user_dev_set_status(dev, status);
301 }
302
303 static uint8_t
304 virtio_user_get_status(struct virtio_hw *hw)
305 {
306         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
307
308         virtio_user_dev_update_status(dev);
309
310         return dev->status;
311 }
312
313 static uint64_t
314 virtio_user_get_features(struct virtio_hw *hw)
315 {
316         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
317
318         /* unmask feature bits defined in vhost user protocol */
319         return dev->device_features & VIRTIO_PMD_SUPPORTED_GUEST_FEATURES;
320 }
321
322 static void
323 virtio_user_set_features(struct virtio_hw *hw, uint64_t features)
324 {
325         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
326
327         dev->features = features & dev->device_features;
328 }
329
330 static uint8_t
331 virtio_user_get_isr(struct virtio_hw *hw __rte_unused)
332 {
333         /* rxq interrupts and config interrupt are separated in virtio-user,
334          * here we only report config change.
335          */
336         return VIRTIO_PCI_ISR_CONFIG;
337 }
338
339 static uint16_t
340 virtio_user_set_config_irq(struct virtio_hw *hw __rte_unused,
341                     uint16_t vec __rte_unused)
342 {
343         return 0;
344 }
345
346 static uint16_t
347 virtio_user_set_queue_irq(struct virtio_hw *hw __rte_unused,
348                           struct virtqueue *vq __rte_unused,
349                           uint16_t vec)
350 {
351         /* pretend we have done that */
352         return vec;
353 }
354
355 /* This function is to get the queue size, aka, number of descs, of a specified
356  * queue. Different with the VHOST_USER_GET_QUEUE_NUM, which is used to get the
357  * max supported queues.
358  */
359 static uint16_t
360 virtio_user_get_queue_num(struct virtio_hw *hw, uint16_t queue_id __rte_unused)
361 {
362         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
363
364         /* Currently, each queue has same queue size */
365         return dev->queue_size;
366 }
367
368 static void
369 virtio_user_setup_queue_packed(struct virtqueue *vq,
370                                struct virtio_user_dev *dev)
371 {
372         uint16_t queue_idx = vq->vq_queue_index;
373         struct vring_packed *vring;
374         uint64_t desc_addr;
375         uint64_t avail_addr;
376         uint64_t used_addr;
377         uint16_t i;
378
379         vring  = &dev->packed_vrings[queue_idx];
380         desc_addr = (uintptr_t)vq->vq_ring_virt_mem;
381         avail_addr = desc_addr + vq->vq_nentries *
382                 sizeof(struct vring_packed_desc);
383         used_addr = RTE_ALIGN_CEIL(avail_addr +
384                            sizeof(struct vring_packed_desc_event),
385                            VIRTIO_PCI_VRING_ALIGN);
386         vring->num = vq->vq_nentries;
387         vring->desc = (void *)(uintptr_t)desc_addr;
388         vring->driver = (void *)(uintptr_t)avail_addr;
389         vring->device = (void *)(uintptr_t)used_addr;
390         dev->packed_queues[queue_idx].avail_wrap_counter = true;
391         dev->packed_queues[queue_idx].used_wrap_counter = true;
392
393         for (i = 0; i < vring->num; i++)
394                 vring->desc[i].flags = 0;
395 }
396
397 static void
398 virtio_user_setup_queue_split(struct virtqueue *vq, struct virtio_user_dev *dev)
399 {
400         uint16_t queue_idx = vq->vq_queue_index;
401         uint64_t desc_addr, avail_addr, used_addr;
402
403         desc_addr = (uintptr_t)vq->vq_ring_virt_mem;
404         avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc);
405         used_addr = RTE_ALIGN_CEIL(avail_addr + offsetof(struct vring_avail,
406                                                          ring[vq->vq_nentries]),
407                                    VIRTIO_PCI_VRING_ALIGN);
408
409         dev->vrings[queue_idx].num = vq->vq_nentries;
410         dev->vrings[queue_idx].desc = (void *)(uintptr_t)desc_addr;
411         dev->vrings[queue_idx].avail = (void *)(uintptr_t)avail_addr;
412         dev->vrings[queue_idx].used = (void *)(uintptr_t)used_addr;
413 }
414
415 static int
416 virtio_user_setup_queue(struct virtio_hw *hw, struct virtqueue *vq)
417 {
418         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
419
420         if (vtpci_packed_queue(hw))
421                 virtio_user_setup_queue_packed(vq, dev);
422         else
423                 virtio_user_setup_queue_split(vq, dev);
424
425         return 0;
426 }
427
428 static void
429 virtio_user_del_queue(struct virtio_hw *hw, struct virtqueue *vq)
430 {
431         /* For legacy devices, write 0 to VIRTIO_PCI_QUEUE_PFN port, QEMU
432          * correspondingly stops the ioeventfds, and reset the status of
433          * the device.
434          * For modern devices, set queue desc, avail, used in PCI bar to 0,
435          * not see any more behavior in QEMU.
436          *
437          * Here we just care about what information to deliver to vhost-user
438          * or vhost-kernel. So we just close ioeventfd for now.
439          */
440         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
441
442         close(dev->callfds[vq->vq_queue_index]);
443         close(dev->kickfds[vq->vq_queue_index]);
444 }
445
446 static void
447 virtio_user_notify_queue(struct virtio_hw *hw, struct virtqueue *vq)
448 {
449         uint64_t buf = 1;
450         struct virtio_user_dev *dev = virtio_user_get_dev(hw);
451
452         if (hw->cvq && (hw->cvq->vq == vq)) {
453                 if (vtpci_packed_queue(vq->hw))
454                         virtio_user_handle_cq_packed(dev, vq->vq_queue_index);
455                 else
456                         virtio_user_handle_cq(dev, vq->vq_queue_index);
457                 return;
458         }
459
460         if (write(dev->kickfds[vq->vq_queue_index], &buf, sizeof(buf)) < 0)
461                 PMD_DRV_LOG(ERR, "failed to kick backend: %s",
462                             strerror(errno));
463 }
464
465 const struct virtio_pci_ops virtio_user_ops = {
466         .read_dev_cfg   = virtio_user_read_dev_config,
467         .write_dev_cfg  = virtio_user_write_dev_config,
468         .get_status     = virtio_user_get_status,
469         .set_status     = virtio_user_set_status,
470         .get_features   = virtio_user_get_features,
471         .set_features   = virtio_user_set_features,
472         .get_isr        = virtio_user_get_isr,
473         .set_config_irq = virtio_user_set_config_irq,
474         .set_queue_irq  = virtio_user_set_queue_irq,
475         .get_queue_num  = virtio_user_get_queue_num,
476         .setup_queue    = virtio_user_setup_queue,
477         .del_queue      = virtio_user_del_queue,
478         .notify_queue   = virtio_user_notify_queue,
479 };
480
481 static const char *valid_args[] = {
482 #define VIRTIO_USER_ARG_QUEUES_NUM     "queues"
483         VIRTIO_USER_ARG_QUEUES_NUM,
484 #define VIRTIO_USER_ARG_CQ_NUM         "cq"
485         VIRTIO_USER_ARG_CQ_NUM,
486 #define VIRTIO_USER_ARG_MAC            "mac"
487         VIRTIO_USER_ARG_MAC,
488 #define VIRTIO_USER_ARG_PATH           "path"
489         VIRTIO_USER_ARG_PATH,
490 #define VIRTIO_USER_ARG_QUEUE_SIZE     "queue_size"
491         VIRTIO_USER_ARG_QUEUE_SIZE,
492 #define VIRTIO_USER_ARG_INTERFACE_NAME "iface"
493         VIRTIO_USER_ARG_INTERFACE_NAME,
494 #define VIRTIO_USER_ARG_SERVER_MODE    "server"
495         VIRTIO_USER_ARG_SERVER_MODE,
496 #define VIRTIO_USER_ARG_MRG_RXBUF      "mrg_rxbuf"
497         VIRTIO_USER_ARG_MRG_RXBUF,
498 #define VIRTIO_USER_ARG_IN_ORDER       "in_order"
499         VIRTIO_USER_ARG_IN_ORDER,
500 #define VIRTIO_USER_ARG_PACKED_VQ      "packed_vq"
501         VIRTIO_USER_ARG_PACKED_VQ,
502 #define VIRTIO_USER_ARG_SPEED          "speed"
503         VIRTIO_USER_ARG_SPEED,
504 #define VIRTIO_USER_ARG_VECTORIZED     "vectorized"
505         VIRTIO_USER_ARG_VECTORIZED,
506         NULL
507 };
508
509 #define VIRTIO_USER_DEF_CQ_EN   0
510 #define VIRTIO_USER_DEF_Q_NUM   1
511 #define VIRTIO_USER_DEF_Q_SZ    256
512 #define VIRTIO_USER_DEF_SERVER_MODE     0
513
514 static int
515 get_string_arg(const char *key __rte_unused,
516                const char *value, void *extra_args)
517 {
518         if (!value || !extra_args)
519                 return -EINVAL;
520
521         *(char **)extra_args = strdup(value);
522
523         if (!*(char **)extra_args)
524                 return -ENOMEM;
525
526         return 0;
527 }
528
529 static int
530 get_integer_arg(const char *key __rte_unused,
531                 const char *value, void *extra_args)
532 {
533         uint64_t integer = 0;
534         if (!value || !extra_args)
535                 return -EINVAL;
536         errno = 0;
537         integer = strtoull(value, NULL, 0);
538         /* extra_args keeps default value, it should be replaced
539          * only in case of successful parsing of the 'value' arg
540          */
541         if (errno == 0)
542                 *(uint64_t *)extra_args = integer;
543         return -errno;
544 }
545
546 static uint32_t
547 vdpa_dynamic_major_num(void)
548 {
549         FILE *fp;
550         char *line = NULL;
551         size_t size;
552         char name[11];
553         bool found = false;
554         uint32_t num;
555
556         fp = fopen("/proc/devices", "r");
557         if (fp == NULL) {
558                 PMD_INIT_LOG(ERR, "Cannot open /proc/devices: %s",
559                              strerror(errno));
560                 return UNNAMED_MAJOR;
561         }
562
563         while (getline(&line, &size, fp) > 0) {
564                 char *stripped = line + strspn(line, " ");
565                 if ((sscanf(stripped, "%u %10s", &num, name) == 2) &&
566                     (strncmp(name, "vhost-vdpa", 10) == 0)) {
567                         found = true;
568                         break;
569                 }
570         }
571         fclose(fp);
572         return found ? num : UNNAMED_MAJOR;
573 }
574
575 static enum virtio_user_backend_type
576 virtio_user_backend_type(const char *path)
577 {
578         struct stat sb;
579
580         if (stat(path, &sb) == -1) {
581                 if (errno == ENOENT)
582                         return VIRTIO_USER_BACKEND_VHOST_USER;
583
584                 PMD_INIT_LOG(ERR, "Stat fails: %s (%s)\n", path,
585                              strerror(errno));
586                 return VIRTIO_USER_BACKEND_UNKNOWN;
587         }
588
589         if (S_ISSOCK(sb.st_mode)) {
590                 return VIRTIO_USER_BACKEND_VHOST_USER;
591         } else if (S_ISCHR(sb.st_mode)) {
592                 if (major(sb.st_rdev) == MISC_MAJOR)
593                         return VIRTIO_USER_BACKEND_VHOST_KERNEL;
594                 if (major(sb.st_rdev) == vdpa_dynamic_major_num())
595                         return VIRTIO_USER_BACKEND_VHOST_VDPA;
596         }
597         return VIRTIO_USER_BACKEND_UNKNOWN;
598 }
599
600 static struct rte_eth_dev *
601 virtio_user_eth_dev_alloc(struct rte_vdev_device *vdev)
602 {
603         struct rte_eth_dev *eth_dev;
604         struct rte_eth_dev_data *data;
605         struct virtio_hw *hw;
606         struct virtio_user_dev *dev;
607
608         eth_dev = rte_eth_vdev_allocate(vdev, sizeof(*dev));
609         if (!eth_dev) {
610                 PMD_INIT_LOG(ERR, "cannot alloc rte_eth_dev");
611                 return NULL;
612         }
613
614         data = eth_dev->data;
615         dev = eth_dev->data->dev_private;
616         hw = &dev->hw;
617
618         hw->port_id = data->port_id;
619         dev->port_id = data->port_id;
620         virtio_hw_internal[hw->port_id].vtpci_ops = &virtio_user_ops;
621         /*
622          * MSIX is required to enable LSC (see virtio_init_device).
623          * Here just pretend that we support msix.
624          */
625         hw->use_msix = 1;
626         hw->bus_type = VIRTIO_BUS_USER;
627         hw->use_vec_rx = 0;
628         hw->use_vec_tx = 0;
629         hw->use_inorder_rx = 0;
630         hw->use_inorder_tx = 0;
631
632         return eth_dev;
633 }
634
635 static void
636 virtio_user_eth_dev_free(struct rte_eth_dev *eth_dev)
637 {
638         rte_eth_dev_release_port(eth_dev);
639 }
640
641 /* Dev initialization routine. Invoked once for each virtio vdev at
642  * EAL init time, see rte_bus_probe().
643  * Returns 0 on success.
644  */
645 static int
646 virtio_user_pmd_probe(struct rte_vdev_device *vdev)
647 {
648         struct rte_kvargs *kvlist = NULL;
649         struct rte_eth_dev *eth_dev;
650         struct virtio_hw *hw;
651         struct virtio_user_dev *dev;
652         enum virtio_user_backend_type backend_type = VIRTIO_USER_BACKEND_UNKNOWN;
653         uint64_t queues = VIRTIO_USER_DEF_Q_NUM;
654         uint64_t cq = VIRTIO_USER_DEF_CQ_EN;
655         uint64_t queue_size = VIRTIO_USER_DEF_Q_SZ;
656         uint64_t server_mode = VIRTIO_USER_DEF_SERVER_MODE;
657         uint64_t mrg_rxbuf = 1;
658         uint64_t in_order = 1;
659         uint64_t packed_vq = 0;
660         uint64_t vectorized = 0;
661         char *path = NULL;
662         char *ifname = NULL;
663         char *mac_addr = NULL;
664         int ret = -1;
665
666         RTE_BUILD_BUG_ON(offsetof(struct virtio_user_dev, hw) != 0);
667
668         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
669                 const char *name = rte_vdev_device_name(vdev);
670                 eth_dev = rte_eth_dev_attach_secondary(name);
671                 if (!eth_dev) {
672                         PMD_INIT_LOG(ERR, "Failed to probe %s", name);
673                         return -1;
674                 }
675
676                 if (eth_virtio_dev_init(eth_dev) < 0) {
677                         PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails");
678                         rte_eth_dev_release_port(eth_dev);
679                         return -1;
680                 }
681
682                 eth_dev->dev_ops = &virtio_user_secondary_eth_dev_ops;
683                 eth_dev->device = &vdev->device;
684                 rte_eth_dev_probing_finish(eth_dev);
685                 return 0;
686         }
687
688         kvlist = rte_kvargs_parse(rte_vdev_device_args(vdev), valid_args);
689         if (!kvlist) {
690                 PMD_INIT_LOG(ERR, "error when parsing param");
691                 goto end;
692         }
693
694         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_PATH) == 1) {
695                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_PATH,
696                                        &get_string_arg, &path) < 0) {
697                         PMD_INIT_LOG(ERR, "error to parse %s",
698                                      VIRTIO_USER_ARG_PATH);
699                         goto end;
700                 }
701         } else {
702                 PMD_INIT_LOG(ERR, "arg %s is mandatory for virtio_user",
703                              VIRTIO_USER_ARG_PATH);
704                 goto end;
705         }
706
707         backend_type = virtio_user_backend_type(path);
708         if (backend_type == VIRTIO_USER_BACKEND_UNKNOWN) {
709                 PMD_INIT_LOG(ERR,
710                              "unable to determine backend type for path %s",
711                         path);
712                 goto end;
713         }
714         PMD_INIT_LOG(INFO, "Backend type detected: %s",
715                      virtio_user_backend_strings[backend_type]);
716
717         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_INTERFACE_NAME) == 1) {
718                 if (backend_type != VIRTIO_USER_BACKEND_VHOST_KERNEL) {
719                         PMD_INIT_LOG(ERR,
720                                 "arg %s applies only to vhost-kernel backend",
721                                 VIRTIO_USER_ARG_INTERFACE_NAME);
722                         goto end;
723                 }
724
725                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_INTERFACE_NAME,
726                                        &get_string_arg, &ifname) < 0) {
727                         PMD_INIT_LOG(ERR, "error to parse %s",
728                                      VIRTIO_USER_ARG_INTERFACE_NAME);
729                         goto end;
730                 }
731         }
732
733         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_MAC) == 1) {
734                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MAC,
735                                        &get_string_arg, &mac_addr) < 0) {
736                         PMD_INIT_LOG(ERR, "error to parse %s",
737                                      VIRTIO_USER_ARG_MAC);
738                         goto end;
739                 }
740         }
741
742         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE) == 1) {
743                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE,
744                                        &get_integer_arg, &queue_size) < 0) {
745                         PMD_INIT_LOG(ERR, "error to parse %s",
746                                      VIRTIO_USER_ARG_QUEUE_SIZE);
747                         goto end;
748                 }
749         }
750
751         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_QUEUES_NUM) == 1) {
752                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUES_NUM,
753                                        &get_integer_arg, &queues) < 0) {
754                         PMD_INIT_LOG(ERR, "error to parse %s",
755                                      VIRTIO_USER_ARG_QUEUES_NUM);
756                         goto end;
757                 }
758         }
759
760         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_SERVER_MODE) == 1) {
761                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_SERVER_MODE,
762                                        &get_integer_arg, &server_mode) < 0) {
763                         PMD_INIT_LOG(ERR, "error to parse %s",
764                                      VIRTIO_USER_ARG_SERVER_MODE);
765                         goto end;
766                 }
767         }
768
769         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_CQ_NUM) == 1) {
770                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_CQ_NUM,
771                                        &get_integer_arg, &cq) < 0) {
772                         PMD_INIT_LOG(ERR, "error to parse %s",
773                                      VIRTIO_USER_ARG_CQ_NUM);
774                         goto end;
775                 }
776         } else if (queues > 1) {
777                 cq = 1;
778         }
779
780         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_PACKED_VQ) == 1) {
781                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_PACKED_VQ,
782                                        &get_integer_arg, &packed_vq) < 0) {
783                         PMD_INIT_LOG(ERR, "error to parse %s",
784                                      VIRTIO_USER_ARG_PACKED_VQ);
785                         goto end;
786                 }
787         }
788
789         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_VECTORIZED) == 1) {
790                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_VECTORIZED,
791                                        &get_integer_arg, &vectorized) < 0) {
792                         PMD_INIT_LOG(ERR, "error to parse %s",
793                                      VIRTIO_USER_ARG_VECTORIZED);
794                         goto end;
795                 }
796         }
797
798         if (queues > 1 && cq == 0) {
799                 PMD_INIT_LOG(ERR, "multi-q requires ctrl-q");
800                 goto end;
801         }
802
803         if (queues > VIRTIO_MAX_VIRTQUEUE_PAIRS) {
804                 PMD_INIT_LOG(ERR, "arg %s %" PRIu64 " exceeds the limit %u",
805                         VIRTIO_USER_ARG_QUEUES_NUM, queues,
806                         VIRTIO_MAX_VIRTQUEUE_PAIRS);
807                 goto end;
808         }
809
810         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_MRG_RXBUF) == 1) {
811                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MRG_RXBUF,
812                                        &get_integer_arg, &mrg_rxbuf) < 0) {
813                         PMD_INIT_LOG(ERR, "error to parse %s",
814                                      VIRTIO_USER_ARG_MRG_RXBUF);
815                         goto end;
816                 }
817         }
818
819         if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_IN_ORDER) == 1) {
820                 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_IN_ORDER,
821                                        &get_integer_arg, &in_order) < 0) {
822                         PMD_INIT_LOG(ERR, "error to parse %s",
823                                      VIRTIO_USER_ARG_IN_ORDER);
824                         goto end;
825                 }
826         }
827
828         eth_dev = virtio_user_eth_dev_alloc(vdev);
829         if (!eth_dev) {
830                 PMD_INIT_LOG(ERR, "virtio_user fails to alloc device");
831                 goto end;
832         }
833
834         dev = eth_dev->data->dev_private;
835         hw = &dev->hw;
836         if (virtio_user_dev_init(dev, path, queues, cq,
837                          queue_size, mac_addr, &ifname, server_mode,
838                          mrg_rxbuf, in_order, packed_vq, backend_type) < 0) {
839                 PMD_INIT_LOG(ERR, "virtio_user_dev_init fails");
840                 virtio_user_eth_dev_free(eth_dev);
841                 goto end;
842         }
843
844         /* previously called by pci probing for physical dev */
845         if (eth_virtio_dev_init(eth_dev) < 0) {
846                 PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails");
847                 virtio_user_eth_dev_free(eth_dev);
848                 goto end;
849         }
850
851         if (vectorized) {
852                 if (packed_vq) {
853 #if defined(CC_AVX512_SUPPORT) || defined(RTE_ARCH_ARM)
854                         hw->use_vec_rx = 1;
855                         hw->use_vec_tx = 1;
856 #else
857                         PMD_INIT_LOG(INFO,
858                                 "building environment do not support packed ring vectorized");
859 #endif
860                 } else {
861                         hw->use_vec_rx = 1;
862                 }
863         }
864
865         rte_eth_dev_probing_finish(eth_dev);
866         ret = 0;
867
868 end:
869         if (kvlist)
870                 rte_kvargs_free(kvlist);
871         if (path)
872                 free(path);
873         if (mac_addr)
874                 free(mac_addr);
875         if (ifname)
876                 free(ifname);
877         return ret;
878 }
879
880 static int
881 virtio_user_pmd_remove(struct rte_vdev_device *vdev)
882 {
883         const char *name;
884         struct rte_eth_dev *eth_dev;
885
886         if (!vdev)
887                 return -EINVAL;
888
889         name = rte_vdev_device_name(vdev);
890         PMD_DRV_LOG(INFO, "Un-Initializing %s", name);
891         eth_dev = rte_eth_dev_allocated(name);
892         /* Port has already been released by close. */
893         if (!eth_dev)
894                 return 0;
895
896         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
897                 return rte_eth_dev_release_port(eth_dev);
898
899         /* make sure the device is stopped, queues freed */
900         return rte_eth_dev_close(eth_dev->data->port_id);
901 }
902
903 static int virtio_user_pmd_dma_map(struct rte_vdev_device *vdev, void *addr,
904                 uint64_t iova, size_t len)
905 {
906         const char *name;
907         struct rte_eth_dev *eth_dev;
908         struct virtio_user_dev *dev;
909
910         if (!vdev)
911                 return -EINVAL;
912
913         name = rte_vdev_device_name(vdev);
914         eth_dev = rte_eth_dev_allocated(name);
915         /* Port has already been released by close. */
916         if (!eth_dev)
917                 return 0;
918
919         dev = eth_dev->data->dev_private;
920
921         if (dev->ops->dma_map)
922                 return dev->ops->dma_map(dev, addr, iova, len);
923
924         return 0;
925 }
926
927 static int virtio_user_pmd_dma_unmap(struct rte_vdev_device *vdev, void *addr,
928                 uint64_t iova, size_t len)
929 {
930         const char *name;
931         struct rte_eth_dev *eth_dev;
932         struct virtio_user_dev *dev;
933
934         if (!vdev)
935                 return -EINVAL;
936
937         name = rte_vdev_device_name(vdev);
938         eth_dev = rte_eth_dev_allocated(name);
939         /* Port has already been released by close. */
940         if (!eth_dev)
941                 return 0;
942
943         dev = eth_dev->data->dev_private;
944
945         if (dev->ops->dma_unmap)
946                 return dev->ops->dma_unmap(dev, addr, iova, len);
947
948         return 0;
949 }
950
951 static struct rte_vdev_driver virtio_user_driver = {
952         .probe = virtio_user_pmd_probe,
953         .remove = virtio_user_pmd_remove,
954         .dma_map = virtio_user_pmd_dma_map,
955         .dma_unmap = virtio_user_pmd_dma_unmap,
956 };
957
958 RTE_PMD_REGISTER_VDEV(net_virtio_user, virtio_user_driver);
959 RTE_PMD_REGISTER_ALIAS(net_virtio_user, virtio_user);
960 RTE_PMD_REGISTER_PARAM_STRING(net_virtio_user,
961         "path=<path> "
962         "mac=<mac addr> "
963         "cq=<int> "
964         "queue_size=<int> "
965         "queues=<int> "
966         "iface=<string> "
967         "server=<0|1> "
968         "mrg_rxbuf=<0|1> "
969         "in_order=<0|1> "
970         "packed_vq=<0|1> "
971         "speed=<int> "
972         "vectorized=<0|1>");