vdpa/mlx5: avoid kick handling during shutdown

When Qemu suspends a VM, HW notifier is un-mmapped while vCPU thread may
still be active and write notifier through kick socket.

PMD kick handler thread tries to install HW notifier through client
socket. In such case, it will timeout and slow down device close.

This patch skips HW notifier install if VQ or device in middle of
shutdown.

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index 749c9d097cfad8b01124f670fde8e1221cc90ad4..48f20d9ecdb3b7d8dc8070a50bc8f0cbce55641d 100644 (file)
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -252,13 +252,15 @@ mlx5_vdpa_dev_close(int vid)
        }
        mlx5_vdpa_err_event_unset(priv);
        mlx5_vdpa_cqe_event_unset(priv);
-       if (priv->configured)
+       if (priv->state == MLX5_VDPA_STATE_CONFIGURED) {
                ret |= mlx5_vdpa_lm_log(priv);
+               priv->state = MLX5_VDPA_STATE_IN_PROGRESS;
+       }
        mlx5_vdpa_steer_unset(priv);
        mlx5_vdpa_virtqs_release(priv);
        mlx5_vdpa_event_qp_global_release(priv);
        mlx5_vdpa_mem_dereg(priv);
-       priv->configured = 0;
+       priv->state = MLX5_VDPA_STATE_PROBED;
        priv->vid = 0;
        /* The mutex may stay locked after event thread cancel - initiate it. */
        pthread_mutex_init(&priv->vq_config_lock, NULL);
@@ -277,7 +279,8 @@ mlx5_vdpa_dev_config(int vid)
                DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
                return -EINVAL;
        }
-       if (priv->configured && mlx5_vdpa_dev_close(vid)) {
+       if (priv->state == MLX5_VDPA_STATE_CONFIGURED &&
+           mlx5_vdpa_dev_close(vid)) {
                DRV_LOG(ERR, "Failed to reconfigure vid %d.", vid);
                return -1;
        }
@@ -291,7 +294,7 @@ mlx5_vdpa_dev_config(int vid)
                mlx5_vdpa_dev_close(vid);
                return -1;
        }
-       priv->configured = 1;
+       priv->state = MLX5_VDPA_STATE_CONFIGURED;
        DRV_LOG(INFO, "vDPA device %d was configured.", vid);
        return 0;
 }
@@ -373,7 +376,7 @@ mlx5_vdpa_get_stats(struct rte_vdpa_device *vdev, int qid,
                DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
                return -ENODEV;
        }
-       if (!priv->configured) {
+       if (priv->state == MLX5_VDPA_STATE_PROBED) {
                DRV_LOG(ERR, "Device %s was not configured.",
                                vdev->device->name);
                return -ENODATA;
@@ -401,7 +404,7 @@ mlx5_vdpa_reset_stats(struct rte_vdpa_device *vdev, int qid)
                DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
                return -ENODEV;
        }
-       if (!priv->configured) {
+       if (priv->state == MLX5_VDPA_STATE_PROBED) {
                DRV_LOG(ERR, "Device %s was not configured.",
                                vdev->device->name);
                return -ENODATA;
@@ -594,7 +597,7 @@ mlx5_vdpa_dev_remove(struct mlx5_common_device *cdev)
                TAILQ_REMOVE(&priv_list, priv, next);
        pthread_mutex_unlock(&priv_list_lock);
        if (found) {
-               if (priv->configured)
+               if (priv->state == MLX5_VDPA_STATE_CONFIGURED)
                        mlx5_vdpa_dev_close(priv->vid);
                if (priv->var) {
                        mlx5_glue->dv_free_var(priv->var);

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h
index 22617924eac5c6540b250d9024ed1cb2321a58fe..cc83d7cba3d1a613501b574edfa3ac4176a100a5 100644 (file)
--- a/drivers/vdpa/mlx5/mlx5_vdpa.h
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.h
@@ -113,9 +113,15 @@ enum {
        MLX5_VDPA_EVENT_MODE_ONLY_INTERRUPT
 };
 
+enum mlx5_dev_state {
+       MLX5_VDPA_STATE_PROBED = 0,
+       MLX5_VDPA_STATE_CONFIGURED,
+       MLX5_VDPA_STATE_IN_PROGRESS /* Shutting down. */
+};
+
 struct mlx5_vdpa_priv {
        TAILQ_ENTRY(mlx5_vdpa_priv) next;
-       uint8_t configured;
+       enum mlx5_dev_state state;
        pthread_mutex_t vq_config_lock;
        uint64_t no_traffic_counter;
        pthread_t timer_tid;

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
index 2696d54b4125f2c4c1ff778a3e74e4f87d554754..4c34983da4105a3dd7b3cc68480cda69dff8c4f9 100644 (file)
--- a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
@@ -25,6 +25,11 @@ mlx5_vdpa_virtq_kick_handler(void *cb_arg)
        int nbytes;
        int retry;
 
+       if (priv->state != MLX5_VDPA_STATE_CONFIGURED && !virtq->enable) {
+               DRV_LOG(ERR,  "device %d queue %d down, skip kick handling",
+                       priv->vid, virtq->index);
+               return;
+       }
        if (rte_intr_fd_get(virtq->intr_handle) < 0)
                return;
        for (retry = 0; retry < 3; ++retry) {
@@ -43,6 +48,11 @@ mlx5_vdpa_virtq_kick_handler(void *cb_arg)
        if (nbytes < 0)
                return;
        rte_write32(virtq->index, priv->virtq_db_addr);
+       if (priv->state != MLX5_VDPA_STATE_CONFIGURED && !virtq->enable) {
+               DRV_LOG(ERR,  "device %d queue %d down, skip kick handling",
+                       priv->vid, virtq->index);
+               return;
+       }
        if (virtq->notifier_state == MLX5_VDPA_NOTIFIER_STATE_DISABLED) {
                if (rte_vhost_host_notifier_ctrl(priv->vid, virtq->index, true))
                        virtq->notifier_state = MLX5_VDPA_NOTIFIER_STATE_ERR;
@@ -541,7 +551,7 @@ mlx5_vdpa_virtq_enable(struct mlx5_vdpa_priv *priv, int index, int enable)
 
        DRV_LOG(INFO, "Update virtq %d status %sable -> %sable.", index,
                virtq->enable ? "en" : "dis", enable ? "en" : "dis");
-       if (!priv->configured) {
+       if (priv->state == MLX5_VDPA_STATE_PROBED) {
                virtq->enable = !!enable;
                return 0;
        }