From e623e0c6d8a5005f9909165e8a034849fa841ed5 Mon Sep 17 00:00:00 2001 From: Yuanhan Liu Date: Fri, 13 May 2016 07:14:19 +0800 Subject: [PATCH] vhost: add reconnect ability Allow reconnecting on failure by default when: - DPDK app starts first and QEMU (as the server) is not started yet. Without reconnecting, DPDK app would simply fail on vhost-user registration. - QEMU restarts, say due to OS reboot. Without reconnecting, you can't re-establish the connection without restarting DPDK app. This patch make it work well for both above cases. It simply creates a new thread, and keep trying calling "connect()", until it succeeds. The reconnect could be disabled when RTE_VHOST_USER_NO_RECONNECT flag is set. Signed-off-by: Yuanhan Liu --- doc/guides/rel_notes/release_16_07.rst | 7 ++ lib/librte_vhost/rte_virtio_net.h | 1 + lib/librte_vhost/vhost_user/vhost-net-user.c | 103 ++++++++++++++++++- 3 files changed, 107 insertions(+), 4 deletions(-) diff --git a/doc/guides/rel_notes/release_16_07.rst b/doc/guides/rel_notes/release_16_07.rst index 1fab04f43a..d6cefa9fb4 100644 --- a/doc/guides/rel_notes/release_16_07.rst +++ b/doc/guides/rel_notes/release_16_07.rst @@ -63,6 +63,13 @@ New Features that a brand new QEMU version (v2.7 or above) is needed, otherwise, the reconnect won't work. + DPDK vhost-user will also try to reconnect by default when + + * the first connect fails (when QEMU is not started yet) + * the connection is broken (when QEMU restarts) + + It can be turned off if flag ``RTE_VHOST_USER_NO_RECONNECT`` is set. + * **Added AES-CTR support to AESNI MB PMD.** Now AESNI MB PMD supports 128/192/256-bit counter mode AES encryption and diff --git a/lib/librte_vhost/rte_virtio_net.h b/lib/librte_vhost/rte_virtio_net.h index 5f69e78d53..9caa62216b 100644 --- a/lib/librte_vhost/rte_virtio_net.h +++ b/lib/librte_vhost/rte_virtio_net.h @@ -52,6 +52,7 @@ #include #define RTE_VHOST_USER_CLIENT (1ULL << 0) +#define RTE_VHOST_USER_NO_RECONNECT (1ULL << 1) /* Enum for virtqueue management. */ enum {VIRTIO_RXQ, VIRTIO_TXQ, VIRTIO_QNUM}; diff --git a/lib/librte_vhost/vhost_user/vhost-net-user.c b/lib/librte_vhost/vhost_user/vhost-net-user.c index a480f9f672..94f1b92393 100644 --- a/lib/librte_vhost/vhost_user/vhost-net-user.c +++ b/lib/librte_vhost/vhost_user/vhost-net-user.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -60,6 +61,7 @@ struct vhost_user_socket { char *path; int listenfd; bool is_server; + bool reconnect; }; struct vhost_user_connection { @@ -79,6 +81,7 @@ struct vhost_user { static void vhost_user_server_new_connection(int fd, void *data, int *remove); static void vhost_user_msg_handler(int fd, void *dat, int *remove); +static int vhost_user_create_client(struct vhost_user_socket *vsocket); static struct vhost_user vhost_user = { .fdset = { @@ -305,6 +308,8 @@ vhost_user_msg_handler(int connfd, void *dat, int *remove) vid = conn->vid; ret = read_vhost_message(connfd, &msg); if (ret <= 0 || msg.request >= VHOST_USER_MAX) { + struct vhost_user_socket *vsocket = conn->vsocket; + if (ret < 0) RTE_LOG(ERR, VHOST_CONFIG, "vhost read message failed\n"); @@ -320,6 +325,9 @@ vhost_user_msg_handler(int connfd, void *dat, int *remove) free(conn); vhost_destroy_device(vid); + if (vsocket->reconnect) + vhost_user_create_client(vsocket); + return; } @@ -471,6 +479,73 @@ err: return -1; } +struct vhost_user_reconnect { + struct sockaddr_un un; + int fd; + struct vhost_user_socket *vsocket; + + TAILQ_ENTRY(vhost_user_reconnect) next; +}; + +TAILQ_HEAD(vhost_user_reconnect_tailq_list, vhost_user_reconnect); +struct vhost_user_reconnect_list { + struct vhost_user_reconnect_tailq_list head; + pthread_mutex_t mutex; +}; + +static struct vhost_user_reconnect_list reconn_list; +static pthread_t reconn_tid; + +static void * +vhost_user_client_reconnect(void *arg __rte_unused) +{ + struct vhost_user_reconnect *reconn, *next; + + while (1) { + pthread_mutex_lock(&reconn_list.mutex); + + /* + * An equal implementation of TAILQ_FOREACH_SAFE, + * which does not exist on all platforms. + */ + for (reconn = TAILQ_FIRST(&reconn_list.head); + reconn != NULL; reconn = next) { + next = TAILQ_NEXT(reconn, next); + + if (connect(reconn->fd, (struct sockaddr *)&reconn->un, + sizeof(reconn->un)) < 0) + continue; + + RTE_LOG(INFO, VHOST_CONFIG, + "%s: connected\n", reconn->vsocket->path); + vhost_user_add_connection(reconn->fd, reconn->vsocket); + TAILQ_REMOVE(&reconn_list.head, reconn, next); + free(reconn); + } + + pthread_mutex_unlock(&reconn_list.mutex); + sleep(1); + } + + return NULL; +} + +static int +vhost_user_reconnect_init(void) +{ + int ret; + + pthread_mutex_init(&reconn_list.mutex, NULL); + TAILQ_INIT(&reconn_list.head); + + ret = pthread_create(&reconn_tid, NULL, + vhost_user_client_reconnect, NULL); + if (ret < 0) + RTE_LOG(ERR, VHOST_CONFIG, "failed to create reconnect thread"); + + return ret; +} + static int vhost_user_create_client(struct vhost_user_socket *vsocket) { @@ -478,20 +553,35 @@ vhost_user_create_client(struct vhost_user_socket *vsocket) int ret; struct sockaddr_un un; const char *path = vsocket->path; + struct vhost_user_reconnect *reconn; fd = create_unix_socket(path, &un, vsocket->is_server); if (fd < 0) return -1; ret = connect(fd, (struct sockaddr *)&un, sizeof(un)); - if (ret < 0) { - RTE_LOG(ERR, VHOST_CONFIG, "failed to connect to %s: %s\n", - path, strerror(errno)); + if (ret == 0) { + vhost_user_add_connection(fd, vsocket); + return 0; + } + + RTE_LOG(ERR, VHOST_CONFIG, + "failed to connect to %s: %s\n", + path, strerror(errno)); + + if (!vsocket->reconnect) { close(fd); return -1; } - vhost_user_add_connection(fd, vsocket); + RTE_LOG(ERR, VHOST_CONFIG, "%s: reconnecting...\n", path); + reconn = malloc(sizeof(*reconn)); + reconn->un = un; + reconn->fd = fd; + reconn->vsocket = vsocket; + pthread_mutex_lock(&reconn_list.mutex); + TAILQ_INSERT_TAIL(&reconn_list.head, reconn, next); + pthread_mutex_unlock(&reconn_list.mutex); return 0; } @@ -525,6 +615,11 @@ rte_vhost_driver_register(const char *path, uint64_t flags) vsocket->path = strdup(path); if ((flags & RTE_VHOST_USER_CLIENT) != 0) { + vsocket->reconnect = !(flags & RTE_VHOST_USER_NO_RECONNECT); + if (vsocket->reconnect && reconn_tid == 0) { + if (vhost_user_reconnect_init() < 0) + goto out; + } ret = vhost_user_create_client(vsocket); } else { vsocket->is_server = true; -- 2.20.1