eal: remove unused PMD types
[dpdk.git] / drivers / net / vhost / rte_eth_vhost.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright (c) 2016 IGEL Co., Ltd.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of IGEL Co.,Ltd. nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 #include <unistd.h>
34 #include <pthread.h>
35 #include <stdbool.h>
36 #ifdef RTE_LIBRTE_VHOST_NUMA
37 #include <numaif.h>
38 #endif
39
40 #include <rte_mbuf.h>
41 #include <rte_ethdev.h>
42 #include <rte_malloc.h>
43 #include <rte_memcpy.h>
44 #include <rte_vdev.h>
45 #include <rte_kvargs.h>
46 #include <rte_virtio_net.h>
47 #include <rte_spinlock.h>
48
49 #include "rte_eth_vhost.h"
50
51 #define ETH_VHOST_IFACE_ARG             "iface"
52 #define ETH_VHOST_QUEUES_ARG            "queues"
53 #define ETH_VHOST_CLIENT_ARG            "client"
54
55 static const char *drivername = "VHOST PMD";
56
57 static const char *valid_arguments[] = {
58         ETH_VHOST_IFACE_ARG,
59         ETH_VHOST_QUEUES_ARG,
60         ETH_VHOST_CLIENT_ARG,
61         NULL
62 };
63
64 static struct ether_addr base_eth_addr = {
65         .addr_bytes = {
66                 0x56 /* V */,
67                 0x48 /* H */,
68                 0x4F /* O */,
69                 0x53 /* S */,
70                 0x54 /* T */,
71                 0x00
72         }
73 };
74
75 struct vhost_queue {
76         int vid;
77         rte_atomic32_t allow_queuing;
78         rte_atomic32_t while_queuing;
79         struct pmd_internal *internal;
80         struct rte_mempool *mb_pool;
81         uint8_t port;
82         uint16_t virtqueue_id;
83         uint64_t rx_pkts;
84         uint64_t tx_pkts;
85         uint64_t missed_pkts;
86         uint64_t rx_bytes;
87         uint64_t tx_bytes;
88 };
89
90 struct pmd_internal {
91         char *dev_name;
92         char *iface_name;
93         uint16_t max_queues;
94         uint64_t flags;
95
96         volatile uint16_t once;
97 };
98
99 struct internal_list {
100         TAILQ_ENTRY(internal_list) next;
101         struct rte_eth_dev *eth_dev;
102 };
103
104 TAILQ_HEAD(internal_list_head, internal_list);
105 static struct internal_list_head internal_list =
106         TAILQ_HEAD_INITIALIZER(internal_list);
107
108 static pthread_mutex_t internal_list_lock = PTHREAD_MUTEX_INITIALIZER;
109
110 static rte_atomic16_t nb_started_ports;
111 static pthread_t session_th;
112
113 static struct rte_eth_link pmd_link = {
114                 .link_speed = 10000,
115                 .link_duplex = ETH_LINK_FULL_DUPLEX,
116                 .link_status = ETH_LINK_DOWN
117 };
118
119 struct rte_vhost_vring_state {
120         rte_spinlock_t lock;
121
122         bool cur[RTE_MAX_QUEUES_PER_PORT * 2];
123         bool seen[RTE_MAX_QUEUES_PER_PORT * 2];
124         unsigned int index;
125         unsigned int max_vring;
126 };
127
128 static struct rte_vhost_vring_state *vring_states[RTE_MAX_ETHPORTS];
129
130 static uint16_t
131 eth_vhost_rx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs)
132 {
133         struct vhost_queue *r = q;
134         uint16_t i, nb_rx = 0;
135
136         if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0))
137                 return 0;
138
139         rte_atomic32_set(&r->while_queuing, 1);
140
141         if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0))
142                 goto out;
143
144         /* Dequeue packets from guest TX queue */
145         nb_rx = rte_vhost_dequeue_burst(r->vid,
146                         r->virtqueue_id, r->mb_pool, bufs, nb_bufs);
147
148         r->rx_pkts += nb_rx;
149
150         for (i = 0; likely(i < nb_rx); i++) {
151                 bufs[i]->port = r->port;
152                 r->rx_bytes += bufs[i]->pkt_len;
153         }
154
155 out:
156         rte_atomic32_set(&r->while_queuing, 0);
157
158         return nb_rx;
159 }
160
161 static uint16_t
162 eth_vhost_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs)
163 {
164         struct vhost_queue *r = q;
165         uint16_t i, nb_tx = 0;
166
167         if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0))
168                 return 0;
169
170         rte_atomic32_set(&r->while_queuing, 1);
171
172         if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0))
173                 goto out;
174
175         /* Enqueue packets to guest RX queue */
176         nb_tx = rte_vhost_enqueue_burst(r->vid,
177                         r->virtqueue_id, bufs, nb_bufs);
178
179         r->tx_pkts += nb_tx;
180         r->missed_pkts += nb_bufs - nb_tx;
181
182         for (i = 0; likely(i < nb_tx); i++)
183                 r->tx_bytes += bufs[i]->pkt_len;
184
185         for (i = 0; likely(i < nb_tx); i++)
186                 rte_pktmbuf_free(bufs[i]);
187 out:
188         rte_atomic32_set(&r->while_queuing, 0);
189
190         return nb_tx;
191 }
192
193 static int
194 eth_dev_configure(struct rte_eth_dev *dev __rte_unused)
195 {
196         return 0;
197 }
198
199 static inline struct internal_list *
200 find_internal_resource(char *ifname)
201 {
202         int found = 0;
203         struct internal_list *list;
204         struct pmd_internal *internal;
205
206         if (ifname == NULL)
207                 return NULL;
208
209         pthread_mutex_lock(&internal_list_lock);
210
211         TAILQ_FOREACH(list, &internal_list, next) {
212                 internal = list->eth_dev->data->dev_private;
213                 if (!strcmp(internal->iface_name, ifname)) {
214                         found = 1;
215                         break;
216                 }
217         }
218
219         pthread_mutex_unlock(&internal_list_lock);
220
221         if (!found)
222                 return NULL;
223
224         return list;
225 }
226
227 static int
228 new_device(int vid)
229 {
230         struct rte_eth_dev *eth_dev;
231         struct internal_list *list;
232         struct pmd_internal *internal;
233         struct vhost_queue *vq;
234         unsigned i;
235         char ifname[PATH_MAX];
236 #ifdef RTE_LIBRTE_VHOST_NUMA
237         int newnode;
238 #endif
239
240         rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
241         list = find_internal_resource(ifname);
242         if (list == NULL) {
243                 RTE_LOG(INFO, PMD, "Invalid device name: %s\n", ifname);
244                 return -1;
245         }
246
247         eth_dev = list->eth_dev;
248         internal = eth_dev->data->dev_private;
249
250 #ifdef RTE_LIBRTE_VHOST_NUMA
251         newnode = rte_vhost_get_numa_node(vid);
252         if (newnode >= 0)
253                 eth_dev->data->numa_node = newnode;
254 #endif
255
256         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
257                 vq = eth_dev->data->rx_queues[i];
258                 if (vq == NULL)
259                         continue;
260                 vq->vid = vid;
261                 vq->internal = internal;
262                 vq->port = eth_dev->data->port_id;
263         }
264         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
265                 vq = eth_dev->data->tx_queues[i];
266                 if (vq == NULL)
267                         continue;
268                 vq->vid = vid;
269                 vq->internal = internal;
270                 vq->port = eth_dev->data->port_id;
271         }
272
273         for (i = 0; i < rte_vhost_get_queue_num(vid) * VIRTIO_QNUM; i++)
274                 rte_vhost_enable_guest_notification(vid, i, 0);
275
276         eth_dev->data->dev_link.link_status = ETH_LINK_UP;
277
278         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
279                 vq = eth_dev->data->rx_queues[i];
280                 if (vq == NULL)
281                         continue;
282                 rte_atomic32_set(&vq->allow_queuing, 1);
283         }
284         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
285                 vq = eth_dev->data->tx_queues[i];
286                 if (vq == NULL)
287                         continue;
288                 rte_atomic32_set(&vq->allow_queuing, 1);
289         }
290
291         RTE_LOG(INFO, PMD, "New connection established\n");
292
293         _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC);
294
295         return 0;
296 }
297
298 static void
299 destroy_device(int vid)
300 {
301         struct rte_eth_dev *eth_dev;
302         struct vhost_queue *vq;
303         struct internal_list *list;
304         char ifname[PATH_MAX];
305         unsigned i;
306         struct rte_vhost_vring_state *state;
307
308         rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
309         list = find_internal_resource(ifname);
310         if (list == NULL) {
311                 RTE_LOG(ERR, PMD, "Invalid interface name: %s\n", ifname);
312                 return;
313         }
314         eth_dev = list->eth_dev;
315
316         /* Wait until rx/tx_pkt_burst stops accessing vhost device */
317         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
318                 vq = eth_dev->data->rx_queues[i];
319                 if (vq == NULL)
320                         continue;
321                 rte_atomic32_set(&vq->allow_queuing, 0);
322                 while (rte_atomic32_read(&vq->while_queuing))
323                         rte_pause();
324         }
325         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
326                 vq = eth_dev->data->tx_queues[i];
327                 if (vq == NULL)
328                         continue;
329                 rte_atomic32_set(&vq->allow_queuing, 0);
330                 while (rte_atomic32_read(&vq->while_queuing))
331                         rte_pause();
332         }
333
334         eth_dev->data->dev_link.link_status = ETH_LINK_DOWN;
335
336         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
337                 vq = eth_dev->data->rx_queues[i];
338                 if (vq == NULL)
339                         continue;
340                 vq->vid = -1;
341         }
342         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
343                 vq = eth_dev->data->tx_queues[i];
344                 if (vq == NULL)
345                         continue;
346                 vq->vid = -1;
347         }
348
349         state = vring_states[eth_dev->data->port_id];
350         rte_spinlock_lock(&state->lock);
351         for (i = 0; i <= state->max_vring; i++) {
352                 state->cur[i] = false;
353                 state->seen[i] = false;
354         }
355         state->max_vring = 0;
356         rte_spinlock_unlock(&state->lock);
357
358         RTE_LOG(INFO, PMD, "Connection closed\n");
359
360         _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC);
361 }
362
363 static int
364 vring_state_changed(int vid, uint16_t vring, int enable)
365 {
366         struct rte_vhost_vring_state *state;
367         struct rte_eth_dev *eth_dev;
368         struct internal_list *list;
369         char ifname[PATH_MAX];
370
371         rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
372         list = find_internal_resource(ifname);
373         if (list == NULL) {
374                 RTE_LOG(ERR, PMD, "Invalid interface name: %s\n", ifname);
375                 return -1;
376         }
377
378         eth_dev = list->eth_dev;
379         /* won't be NULL */
380         state = vring_states[eth_dev->data->port_id];
381         rte_spinlock_lock(&state->lock);
382         state->cur[vring] = enable;
383         state->max_vring = RTE_MAX(vring, state->max_vring);
384         rte_spinlock_unlock(&state->lock);
385
386         RTE_LOG(INFO, PMD, "vring%u is %s\n",
387                         vring, enable ? "enabled" : "disabled");
388
389         _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_QUEUE_STATE);
390
391         return 0;
392 }
393
394 int
395 rte_eth_vhost_get_queue_event(uint8_t port_id,
396                 struct rte_eth_vhost_queue_event *event)
397 {
398         struct rte_vhost_vring_state *state;
399         unsigned int i;
400         int idx;
401
402         if (port_id >= RTE_MAX_ETHPORTS) {
403                 RTE_LOG(ERR, PMD, "Invalid port id\n");
404                 return -1;
405         }
406
407         state = vring_states[port_id];
408         if (!state) {
409                 RTE_LOG(ERR, PMD, "Unused port\n");
410                 return -1;
411         }
412
413         rte_spinlock_lock(&state->lock);
414         for (i = 0; i <= state->max_vring; i++) {
415                 idx = state->index++ % (state->max_vring + 1);
416
417                 if (state->cur[idx] != state->seen[idx]) {
418                         state->seen[idx] = state->cur[idx];
419                         event->queue_id = idx / 2;
420                         event->rx = idx & 1;
421                         event->enable = state->cur[idx];
422                         rte_spinlock_unlock(&state->lock);
423                         return 0;
424                 }
425         }
426         rte_spinlock_unlock(&state->lock);
427
428         return -1;
429 }
430
431 static void *
432 vhost_driver_session(void *param __rte_unused)
433 {
434         static struct virtio_net_device_ops vhost_ops;
435
436         /* set vhost arguments */
437         vhost_ops.new_device = new_device;
438         vhost_ops.destroy_device = destroy_device;
439         vhost_ops.vring_state_changed = vring_state_changed;
440         if (rte_vhost_driver_callback_register(&vhost_ops) < 0)
441                 RTE_LOG(ERR, PMD, "Can't register callbacks\n");
442
443         /* start event handling */
444         rte_vhost_driver_session_start();
445
446         return NULL;
447 }
448
449 static int
450 vhost_driver_session_start(void)
451 {
452         int ret;
453
454         ret = pthread_create(&session_th,
455                         NULL, vhost_driver_session, NULL);
456         if (ret)
457                 RTE_LOG(ERR, PMD, "Can't create a thread\n");
458
459         return ret;
460 }
461
462 static void
463 vhost_driver_session_stop(void)
464 {
465         int ret;
466
467         ret = pthread_cancel(session_th);
468         if (ret)
469                 RTE_LOG(ERR, PMD, "Can't cancel the thread\n");
470
471         ret = pthread_join(session_th, NULL);
472         if (ret)
473                 RTE_LOG(ERR, PMD, "Can't join the thread\n");
474 }
475
476 static int
477 eth_dev_start(struct rte_eth_dev *dev)
478 {
479         struct pmd_internal *internal = dev->data->dev_private;
480         int ret = 0;
481
482         if (rte_atomic16_cmpset(&internal->once, 0, 1)) {
483                 ret = rte_vhost_driver_register(internal->iface_name,
484                                                 internal->flags);
485                 if (ret)
486                         return ret;
487         }
488
489         /* We need only one message handling thread */
490         if (rte_atomic16_add_return(&nb_started_ports, 1) == 1)
491                 ret = vhost_driver_session_start();
492
493         return ret;
494 }
495
496 static void
497 eth_dev_stop(struct rte_eth_dev *dev)
498 {
499         struct pmd_internal *internal = dev->data->dev_private;
500
501         if (rte_atomic16_cmpset(&internal->once, 1, 0))
502                 rte_vhost_driver_unregister(internal->iface_name);
503
504         if (rte_atomic16_sub_return(&nb_started_ports, 1) == 0)
505                 vhost_driver_session_stop();
506 }
507
508 static int
509 eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
510                    uint16_t nb_rx_desc __rte_unused,
511                    unsigned int socket_id,
512                    const struct rte_eth_rxconf *rx_conf __rte_unused,
513                    struct rte_mempool *mb_pool)
514 {
515         struct vhost_queue *vq;
516
517         vq = rte_zmalloc_socket(NULL, sizeof(struct vhost_queue),
518                         RTE_CACHE_LINE_SIZE, socket_id);
519         if (vq == NULL) {
520                 RTE_LOG(ERR, PMD, "Failed to allocate memory for rx queue\n");
521                 return -ENOMEM;
522         }
523
524         vq->mb_pool = mb_pool;
525         vq->virtqueue_id = rx_queue_id * VIRTIO_QNUM + VIRTIO_TXQ;
526         dev->data->rx_queues[rx_queue_id] = vq;
527
528         return 0;
529 }
530
531 static int
532 eth_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
533                    uint16_t nb_tx_desc __rte_unused,
534                    unsigned int socket_id,
535                    const struct rte_eth_txconf *tx_conf __rte_unused)
536 {
537         struct vhost_queue *vq;
538
539         vq = rte_zmalloc_socket(NULL, sizeof(struct vhost_queue),
540                         RTE_CACHE_LINE_SIZE, socket_id);
541         if (vq == NULL) {
542                 RTE_LOG(ERR, PMD, "Failed to allocate memory for tx queue\n");
543                 return -ENOMEM;
544         }
545
546         vq->virtqueue_id = tx_queue_id * VIRTIO_QNUM + VIRTIO_RXQ;
547         dev->data->tx_queues[tx_queue_id] = vq;
548
549         return 0;
550 }
551
552 static void
553 eth_dev_info(struct rte_eth_dev *dev,
554              struct rte_eth_dev_info *dev_info)
555 {
556         struct pmd_internal *internal;
557
558         internal = dev->data->dev_private;
559         if (internal == NULL) {
560                 RTE_LOG(ERR, PMD, "Invalid device specified\n");
561                 return;
562         }
563
564         dev_info->driver_name = drivername;
565         dev_info->max_mac_addrs = 1;
566         dev_info->max_rx_pktlen = (uint32_t)-1;
567         dev_info->max_rx_queues = internal->max_queues;
568         dev_info->max_tx_queues = internal->max_queues;
569         dev_info->min_rx_bufsize = 0;
570 }
571
572 static void
573 eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
574 {
575         unsigned i;
576         unsigned long rx_total = 0, tx_total = 0, tx_missed_total = 0;
577         unsigned long rx_total_bytes = 0, tx_total_bytes = 0;
578         struct vhost_queue *vq;
579
580         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS &&
581                         i < dev->data->nb_rx_queues; i++) {
582                 if (dev->data->rx_queues[i] == NULL)
583                         continue;
584                 vq = dev->data->rx_queues[i];
585                 stats->q_ipackets[i] = vq->rx_pkts;
586                 rx_total += stats->q_ipackets[i];
587
588                 stats->q_ibytes[i] = vq->rx_bytes;
589                 rx_total_bytes += stats->q_ibytes[i];
590         }
591
592         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS &&
593                         i < dev->data->nb_tx_queues; i++) {
594                 if (dev->data->tx_queues[i] == NULL)
595                         continue;
596                 vq = dev->data->tx_queues[i];
597                 stats->q_opackets[i] = vq->tx_pkts;
598                 tx_missed_total += vq->missed_pkts;
599                 tx_total += stats->q_opackets[i];
600
601                 stats->q_obytes[i] = vq->tx_bytes;
602                 tx_total_bytes += stats->q_obytes[i];
603         }
604
605         stats->ipackets = rx_total;
606         stats->opackets = tx_total;
607         stats->oerrors = tx_missed_total;
608         stats->ibytes = rx_total_bytes;
609         stats->obytes = tx_total_bytes;
610 }
611
612 static void
613 eth_stats_reset(struct rte_eth_dev *dev)
614 {
615         struct vhost_queue *vq;
616         unsigned i;
617
618         for (i = 0; i < dev->data->nb_rx_queues; i++) {
619                 if (dev->data->rx_queues[i] == NULL)
620                         continue;
621                 vq = dev->data->rx_queues[i];
622                 vq->rx_pkts = 0;
623                 vq->rx_bytes = 0;
624         }
625         for (i = 0; i < dev->data->nb_tx_queues; i++) {
626                 if (dev->data->tx_queues[i] == NULL)
627                         continue;
628                 vq = dev->data->tx_queues[i];
629                 vq->tx_pkts = 0;
630                 vq->tx_bytes = 0;
631                 vq->missed_pkts = 0;
632         }
633 }
634
635 static void
636 eth_queue_release(void *q)
637 {
638         rte_free(q);
639 }
640
641 static int
642 eth_link_update(struct rte_eth_dev *dev __rte_unused,
643                 int wait_to_complete __rte_unused)
644 {
645         return 0;
646 }
647
648 /**
649  * Disable features in feature_mask. Returns 0 on success.
650  */
651 int
652 rte_eth_vhost_feature_disable(uint64_t feature_mask)
653 {
654         return rte_vhost_feature_disable(feature_mask);
655 }
656
657 /**
658  * Enable features in feature_mask. Returns 0 on success.
659  */
660 int
661 rte_eth_vhost_feature_enable(uint64_t feature_mask)
662 {
663         return rte_vhost_feature_enable(feature_mask);
664 }
665
666 /* Returns currently supported vhost features */
667 uint64_t
668 rte_eth_vhost_feature_get(void)
669 {
670         return rte_vhost_feature_get();
671 }
672
673 static const struct eth_dev_ops ops = {
674         .dev_start = eth_dev_start,
675         .dev_stop = eth_dev_stop,
676         .dev_configure = eth_dev_configure,
677         .dev_infos_get = eth_dev_info,
678         .rx_queue_setup = eth_rx_queue_setup,
679         .tx_queue_setup = eth_tx_queue_setup,
680         .rx_queue_release = eth_queue_release,
681         .tx_queue_release = eth_queue_release,
682         .link_update = eth_link_update,
683         .stats_get = eth_stats_get,
684         .stats_reset = eth_stats_reset,
685 };
686
687 static int
688 eth_dev_vhost_create(const char *name, char *iface_name, int16_t queues,
689                      const unsigned numa_node, uint64_t flags)
690 {
691         struct rte_eth_dev_data *data = NULL;
692         struct pmd_internal *internal = NULL;
693         struct rte_eth_dev *eth_dev = NULL;
694         struct ether_addr *eth_addr = NULL;
695         struct rte_vhost_vring_state *vring_state = NULL;
696         struct internal_list *list = NULL;
697
698         RTE_LOG(INFO, PMD, "Creating VHOST-USER backend on numa socket %u\n",
699                 numa_node);
700
701         /* now do all data allocation - for eth_dev structure, dummy pci driver
702          * and internal (private) data
703          */
704         data = rte_zmalloc_socket(name, sizeof(*data), 0, numa_node);
705         if (data == NULL)
706                 goto error;
707
708         internal = rte_zmalloc_socket(name, sizeof(*internal), 0, numa_node);
709         if (internal == NULL)
710                 goto error;
711
712         list = rte_zmalloc_socket(name, sizeof(*list), 0, numa_node);
713         if (list == NULL)
714                 goto error;
715
716         /* reserve an ethdev entry */
717         eth_dev = rte_eth_dev_allocate(name);
718         if (eth_dev == NULL)
719                 goto error;
720
721         eth_addr = rte_zmalloc_socket(name, sizeof(*eth_addr), 0, numa_node);
722         if (eth_addr == NULL)
723                 goto error;
724         *eth_addr = base_eth_addr;
725         eth_addr->addr_bytes[5] = eth_dev->data->port_id;
726
727         vring_state = rte_zmalloc_socket(name,
728                         sizeof(*vring_state), 0, numa_node);
729         if (vring_state == NULL)
730                 goto error;
731
732         TAILQ_INIT(&eth_dev->link_intr_cbs);
733
734         /* now put it all together
735          * - store queue data in internal,
736          * - store numa_node info in ethdev data
737          * - point eth_dev_data to internals
738          * - and point eth_dev structure to new eth_dev_data structure
739          */
740         internal->dev_name = strdup(name);
741         if (internal->dev_name == NULL)
742                 goto error;
743         internal->iface_name = strdup(iface_name);
744         if (internal->iface_name == NULL)
745                 goto error;
746         internal->flags = flags;
747
748         list->eth_dev = eth_dev;
749         pthread_mutex_lock(&internal_list_lock);
750         TAILQ_INSERT_TAIL(&internal_list, list, next);
751         pthread_mutex_unlock(&internal_list_lock);
752
753         rte_spinlock_init(&vring_state->lock);
754         vring_states[eth_dev->data->port_id] = vring_state;
755
756         data->dev_private = internal;
757         data->port_id = eth_dev->data->port_id;
758         memmove(data->name, eth_dev->data->name, sizeof(data->name));
759         data->nb_rx_queues = queues;
760         data->nb_tx_queues = queues;
761         internal->max_queues = queues;
762         data->dev_link = pmd_link;
763         data->mac_addrs = eth_addr;
764
765         /* We'll replace the 'data' originally allocated by eth_dev. So the
766          * vhost PMD resources won't be shared between multi processes.
767          */
768         eth_dev->data = data;
769         eth_dev->dev_ops = &ops;
770         eth_dev->driver = NULL;
771         data->dev_flags =
772                 RTE_ETH_DEV_DETACHABLE | RTE_ETH_DEV_INTR_LSC;
773         data->kdrv = RTE_KDRV_NONE;
774         data->drv_name = internal->dev_name;
775         data->numa_node = numa_node;
776
777         /* finally assign rx and tx ops */
778         eth_dev->rx_pkt_burst = eth_vhost_rx;
779         eth_dev->tx_pkt_burst = eth_vhost_tx;
780
781         return data->port_id;
782
783 error:
784         if (internal)
785                 free(internal->dev_name);
786         rte_free(vring_state);
787         rte_free(eth_addr);
788         if (eth_dev)
789                 rte_eth_dev_release_port(eth_dev);
790         rte_free(internal);
791         rte_free(list);
792         rte_free(data);
793
794         return -1;
795 }
796
797 static inline int
798 open_iface(const char *key __rte_unused, const char *value, void *extra_args)
799 {
800         const char **iface_name = extra_args;
801
802         if (value == NULL)
803                 return -1;
804
805         *iface_name = value;
806
807         return 0;
808 }
809
810 static inline int
811 open_int(const char *key __rte_unused, const char *value, void *extra_args)
812 {
813         uint16_t *n = extra_args;
814
815         if (value == NULL || extra_args == NULL)
816                 return -EINVAL;
817
818         *n = (uint16_t)strtoul(value, NULL, 0);
819         if (*n == USHRT_MAX && errno == ERANGE)
820                 return -1;
821
822         return 0;
823 }
824
825 static int
826 rte_pmd_vhost_devinit(const char *name, const char *params)
827 {
828         struct rte_kvargs *kvlist = NULL;
829         int ret = 0;
830         char *iface_name;
831         uint16_t queues;
832         uint64_t flags = 0;
833         int client_mode = 0;
834
835         RTE_LOG(INFO, PMD, "Initializing pmd_vhost for %s\n", name);
836
837         kvlist = rte_kvargs_parse(params, valid_arguments);
838         if (kvlist == NULL)
839                 return -1;
840
841         if (rte_kvargs_count(kvlist, ETH_VHOST_IFACE_ARG) == 1) {
842                 ret = rte_kvargs_process(kvlist, ETH_VHOST_IFACE_ARG,
843                                          &open_iface, &iface_name);
844                 if (ret < 0)
845                         goto out_free;
846         } else {
847                 ret = -1;
848                 goto out_free;
849         }
850
851         if (rte_kvargs_count(kvlist, ETH_VHOST_QUEUES_ARG) == 1) {
852                 ret = rte_kvargs_process(kvlist, ETH_VHOST_QUEUES_ARG,
853                                          &open_int, &queues);
854                 if (ret < 0 || queues > RTE_MAX_QUEUES_PER_PORT)
855                         goto out_free;
856
857         } else
858                 queues = 1;
859
860         if (rte_kvargs_count(kvlist, ETH_VHOST_CLIENT_ARG) == 1) {
861                 ret = rte_kvargs_process(kvlist, ETH_VHOST_CLIENT_ARG,
862                                          &open_int, &client_mode);
863                 if (ret < 0)
864                         goto out_free;
865
866                 if (client_mode)
867                         flags |= RTE_VHOST_USER_CLIENT;
868         }
869
870         eth_dev_vhost_create(name, iface_name, queues, rte_socket_id(), flags);
871
872 out_free:
873         rte_kvargs_free(kvlist);
874         return ret;
875 }
876
877 static int
878 rte_pmd_vhost_devuninit(const char *name)
879 {
880         struct rte_eth_dev *eth_dev = NULL;
881         struct pmd_internal *internal;
882         struct internal_list *list;
883         unsigned int i;
884
885         RTE_LOG(INFO, PMD, "Un-Initializing pmd_vhost for %s\n", name);
886
887         /* find an ethdev entry */
888         eth_dev = rte_eth_dev_allocated(name);
889         if (eth_dev == NULL)
890                 return -ENODEV;
891
892         internal = eth_dev->data->dev_private;
893         if (internal == NULL)
894                 return -ENODEV;
895
896         list = find_internal_resource(internal->iface_name);
897         if (list == NULL)
898                 return -ENODEV;
899
900         pthread_mutex_lock(&internal_list_lock);
901         TAILQ_REMOVE(&internal_list, list, next);
902         pthread_mutex_unlock(&internal_list_lock);
903         rte_free(list);
904
905         eth_dev_stop(eth_dev);
906
907         rte_free(vring_states[eth_dev->data->port_id]);
908         vring_states[eth_dev->data->port_id] = NULL;
909
910         free(internal->dev_name);
911         free(internal->iface_name);
912
913         for (i = 0; i < eth_dev->data->nb_rx_queues; i++)
914                 rte_free(eth_dev->data->rx_queues[i]);
915         for (i = 0; i < eth_dev->data->nb_tx_queues; i++)
916                 rte_free(eth_dev->data->tx_queues[i]);
917
918         rte_free(eth_dev->data->mac_addrs);
919         rte_free(eth_dev->data);
920         rte_free(internal);
921
922         rte_eth_dev_release_port(eth_dev);
923
924         return 0;
925 }
926
927 static struct rte_vdev_driver pmd_vhost_drv = {
928         .init = rte_pmd_vhost_devinit,
929         .uninit = rte_pmd_vhost_devuninit,
930 };
931
932 DRIVER_REGISTER_VDEV(net_vhost, pmd_vhost_drv);
933 DRIVER_REGISTER_PARAM_STRING(net_vhost,
934         "iface=<ifc> "
935         "queues=<int>");