X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fvdpa%2Fmlx5%2Fmlx5_vdpa_virtq.c;h=63b6f4472543798bbe9aee8f6c414005ecb0a610;hb=057f7d20849907f55bcb8a7b51014805ffa26265;hp=6637ba15039cddb4f81407f9384a0f40bd9bb8aa;hpb=24969c7b6224afc48751d94fc0152fca8b6645b1;p=dpdk.git diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c index 6637ba1503..63b6f44725 100644 --- a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +++ b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c @@ -24,13 +24,17 @@ mlx5_vdpa_virtq_kick_handler(void *cb_arg) int nbytes; int retry; + pthread_mutex_lock(&virtq->virtq_lock); if (priv->state != MLX5_VDPA_STATE_CONFIGURED && !virtq->enable) { + pthread_mutex_unlock(&virtq->virtq_lock); DRV_LOG(ERR, "device %d queue %d down, skip kick handling", priv->vid, virtq->index); return; } - if (rte_intr_fd_get(virtq->intr_handle) < 0) + if (rte_intr_fd_get(virtq->intr_handle) < 0) { + pthread_mutex_unlock(&virtq->virtq_lock); return; + } for (retry = 0; retry < 3; ++retry) { nbytes = read(rte_intr_fd_get(virtq->intr_handle), &buf, 8); @@ -44,9 +48,14 @@ mlx5_vdpa_virtq_kick_handler(void *cb_arg) } break; } - if (nbytes < 0) + if (nbytes < 0) { + pthread_mutex_unlock(&virtq->virtq_lock); return; + } + rte_spinlock_lock(&priv->db_lock); rte_write32(virtq->index, priv->virtq_db_addr); + rte_spinlock_unlock(&priv->db_lock); + pthread_mutex_unlock(&virtq->virtq_lock); if (priv->state != MLX5_VDPA_STATE_CONFIGURED && !virtq->enable) { DRV_LOG(ERR, "device %d queue %d down, skip kick handling", priv->vid, virtq->index); @@ -66,6 +75,33 @@ mlx5_vdpa_virtq_kick_handler(void *cb_arg) DRV_LOG(DEBUG, "Ring virtq %u doorbell.", virtq->index); } +/* Virtq must be locked before calling this function. */ +static void +mlx5_vdpa_virtq_unregister_intr_handle(struct mlx5_vdpa_virtq *virtq) +{ + int ret = -EAGAIN; + + if (!virtq->intr_handle) + return; + if (rte_intr_fd_get(virtq->intr_handle) >= 0) { + while (ret == -EAGAIN) { + ret = rte_intr_callback_unregister(virtq->intr_handle, + mlx5_vdpa_virtq_kick_handler, virtq); + if (ret == -EAGAIN) { + DRV_LOG(DEBUG, "Try again to unregister fd %d of virtq %hu interrupt", + rte_intr_fd_get(virtq->intr_handle), + virtq->index); + pthread_mutex_unlock(&virtq->virtq_lock); + usleep(MLX5_VDPA_INTR_RETRIES_USEC); + pthread_mutex_lock(&virtq->virtq_lock); + } + } + (void)rte_intr_fd_set(virtq->intr_handle, -1); + } + rte_intr_instance_free(virtq->intr_handle); + virtq->intr_handle = NULL; +} + /* Release cached VQ resources. */ void mlx5_vdpa_virtqs_cleanup(struct mlx5_vdpa_priv *priv) @@ -75,6 +111,8 @@ mlx5_vdpa_virtqs_cleanup(struct mlx5_vdpa_priv *priv) for (i = 0; i < priv->caps.max_num_virtio_queues; i++) { struct mlx5_vdpa_virtq *virtq = &priv->virtqs[i]; + pthread_mutex_lock(&virtq->virtq_lock); + virtq->configured = 0; for (j = 0; j < RTE_DIM(virtq->umems); ++j) { if (virtq->umems[j].obj) { claim_zero(mlx5_glue->devx_umem_dereg @@ -89,33 +127,23 @@ mlx5_vdpa_virtqs_cleanup(struct mlx5_vdpa_priv *priv) } if (virtq->eqp.fw_qp) mlx5_vdpa_event_qp_destroy(&virtq->eqp); + pthread_mutex_unlock(&virtq->virtq_lock); } } + static int mlx5_vdpa_virtq_unset(struct mlx5_vdpa_virtq *virtq) { int ret = -EAGAIN; - if (rte_intr_fd_get(virtq->intr_handle) >= 0) { - while (ret == -EAGAIN) { - ret = rte_intr_callback_unregister(virtq->intr_handle, - mlx5_vdpa_virtq_kick_handler, virtq); - if (ret == -EAGAIN) { - DRV_LOG(DEBUG, "Try again to unregister fd %d of virtq %hu interrupt", - rte_intr_fd_get(virtq->intr_handle), - virtq->index); - usleep(MLX5_VDPA_INTR_RETRIES_USEC); - } - } - rte_intr_fd_set(virtq->intr_handle, -1); - } - rte_intr_instance_free(virtq->intr_handle); - if (virtq->virtq) { + mlx5_vdpa_virtq_unregister_intr_handle(virtq); + if (virtq->configured) { ret = mlx5_vdpa_virtq_stop(virtq->priv, virtq->index); if (ret) DRV_LOG(WARNING, "Failed to stop virtq %d.", virtq->index); + virtq->configured = 0; claim_zero(mlx5_devx_cmd_destroy(virtq->virtq)); } virtq->virtq = NULL; @@ -126,10 +154,15 @@ mlx5_vdpa_virtq_unset(struct mlx5_vdpa_virtq *virtq) void mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv) { + struct mlx5_vdpa_virtq *virtq; int i; - for (i = 0; i < priv->nr_virtqs; i++) - mlx5_vdpa_virtq_unset(&priv->virtqs[i]); + for (i = 0; i < priv->nr_virtqs; i++) { + virtq = &priv->virtqs[i]; + pthread_mutex_lock(&virtq->virtq_lock); + mlx5_vdpa_virtq_unset(virtq); + pthread_mutex_unlock(&virtq->virtq_lock); + } priv->features = 0; priv->nr_virtqs = 0; } @@ -138,7 +171,7 @@ int mlx5_vdpa_virtq_modify(struct mlx5_vdpa_virtq *virtq, int state) { struct mlx5_devx_virtq_attr attr = { - .type = MLX5_VIRTQ_MODIFY_TYPE_STATE, + .mod_fields_bitmap = MLX5_VIRTQ_MODIFY_TYPE_STATE, .state = state ? MLX5_VIRTQ_STATE_RDY : MLX5_VIRTQ_STATE_SUSPEND, .queue_index = virtq->index, @@ -153,7 +186,7 @@ mlx5_vdpa_virtq_stop(struct mlx5_vdpa_priv *priv, int index) struct mlx5_vdpa_virtq *virtq = &priv->virtqs[index]; int ret; - if (virtq->stopped) + if (virtq->stopped || !virtq->configured) return 0; ret = mlx5_vdpa_virtq_modify(virtq, 0); if (ret) @@ -209,51 +242,54 @@ mlx5_vdpa_hva_to_gpa(struct rte_vhost_memory *mem, uint64_t hva) } static int -mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv, int index) +mlx5_vdpa_virtq_sub_objs_prepare(struct mlx5_vdpa_priv *priv, + struct mlx5_devx_virtq_attr *attr, + struct rte_vhost_vring *vq, int index) { struct mlx5_vdpa_virtq *virtq = &priv->virtqs[index]; - struct rte_vhost_vring vq; - struct mlx5_devx_virtq_attr attr = {0}; uint64_t gpa; int ret; unsigned int i; - uint16_t last_avail_idx; - uint16_t last_used_idx; - uint16_t event_num = MLX5_EVENT_TYPE_OBJECT_CHANGE; - uint64_t cookie; - - ret = rte_vhost_get_vhost_vring(priv->vid, index, &vq); - if (ret) - return -1; - if (vq.size == 0) - return 0; - virtq->index = index; - virtq->vq_size = vq.size; - attr.tso_ipv4 = !!(priv->features & (1ULL << VIRTIO_NET_F_HOST_TSO4)); - attr.tso_ipv6 = !!(priv->features & (1ULL << VIRTIO_NET_F_HOST_TSO6)); - attr.tx_csum = !!(priv->features & (1ULL << VIRTIO_NET_F_CSUM)); - attr.rx_csum = !!(priv->features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)); - attr.virtio_version_1_0 = !!(priv->features & (1ULL << - VIRTIO_F_VERSION_1)); - attr.type = (priv->features & (1ULL << VIRTIO_F_RING_PACKED)) ? + uint16_t last_avail_idx = 0; + uint16_t last_used_idx = 0; + + if (virtq->virtq) + attr->mod_fields_bitmap = MLX5_VIRTQ_MODIFY_TYPE_STATE | + MLX5_VIRTQ_MODIFY_TYPE_ADDR | + MLX5_VIRTQ_MODIFY_TYPE_HW_AVAILABLE_INDEX | + MLX5_VIRTQ_MODIFY_TYPE_HW_USED_INDEX | + MLX5_VIRTQ_MODIFY_TYPE_VERSION_1_0 | + MLX5_VIRTQ_MODIFY_TYPE_Q_TYPE | + MLX5_VIRTQ_MODIFY_TYPE_Q_MKEY | + MLX5_VIRTQ_MODIFY_TYPE_QUEUE_FEATURE_BIT_MASK | + MLX5_VIRTQ_MODIFY_TYPE_EVENT_MODE; + attr->tso_ipv4 = !!(priv->features & (1ULL << VIRTIO_NET_F_HOST_TSO4)); + attr->tso_ipv6 = !!(priv->features & (1ULL << VIRTIO_NET_F_HOST_TSO6)); + attr->tx_csum = !!(priv->features & (1ULL << VIRTIO_NET_F_CSUM)); + attr->rx_csum = !!(priv->features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)); + attr->virtio_version_1_0 = + !!(priv->features & (1ULL << VIRTIO_F_VERSION_1)); + attr->q_type = + (priv->features & (1ULL << VIRTIO_F_RING_PACKED)) ? MLX5_VIRTQ_TYPE_PACKED : MLX5_VIRTQ_TYPE_SPLIT; /* * No need event QPs creation when the guest in poll mode or when the * capability allows it. */ - attr.event_mode = vq.callfd != -1 || !(priv->caps.event_mode & (1 << - MLX5_VIRTQ_EVENT_MODE_NO_MSIX)) ? - MLX5_VIRTQ_EVENT_MODE_QP : - MLX5_VIRTQ_EVENT_MODE_NO_MSIX; - if (attr.event_mode == MLX5_VIRTQ_EVENT_MODE_QP) { - ret = mlx5_vdpa_event_qp_prepare(priv, vq.size, vq.callfd, - &virtq->eqp); + attr->event_mode = vq->callfd != -1 || + !(priv->caps.event_mode & (1 << MLX5_VIRTQ_EVENT_MODE_NO_MSIX)) ? + MLX5_VIRTQ_EVENT_MODE_QP : MLX5_VIRTQ_EVENT_MODE_NO_MSIX; + if (attr->event_mode == MLX5_VIRTQ_EVENT_MODE_QP) { + ret = mlx5_vdpa_event_qp_prepare(priv, + vq->size, vq->callfd, virtq); if (ret) { - DRV_LOG(ERR, "Failed to create event QPs for virtq %d.", + DRV_LOG(ERR, + "Failed to create event QPs for virtq %d.", index); return -1; } - attr.qp_id = virtq->eqp.fw_qp->id; + attr->mod_fields_bitmap |= MLX5_VIRTQ_MODIFY_TYPE_EVENT_MODE; + attr->qp_id = virtq->eqp.fw_qp->id; } else { DRV_LOG(INFO, "Virtq %d is, for sure, working by poll mode, no" " need event QPs and event mechanism.", index); @@ -265,77 +301,82 @@ mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv, int index) if (!virtq->counters) { DRV_LOG(ERR, "Failed to create virtq couners for virtq" " %d.", index); - goto error; + return -1; } - attr.counters_obj_id = virtq->counters->id; + attr->counters_obj_id = virtq->counters->id; } /* Setup 3 UMEMs for each virtq. */ - for (i = 0; i < RTE_DIM(virtq->umems); ++i) { - uint32_t size; - void *buf; - struct mlx5dv_devx_umem *obj; - - size = priv->caps.umems[i].a * vq.size + priv->caps.umems[i].b; - if (virtq->umems[i].size == size && - virtq->umems[i].obj != NULL) { - /* Reuse registered memory. */ - memset(virtq->umems[i].buf, 0, size); - goto reuse; - } - if (virtq->umems[i].obj) - claim_zero(mlx5_glue->devx_umem_dereg + if (virtq->virtq) { + for (i = 0; i < RTE_DIM(virtq->umems); ++i) { + uint32_t size; + void *buf; + struct mlx5dv_devx_umem *obj; + + size = + priv->caps.umems[i].a * vq->size + priv->caps.umems[i].b; + if (virtq->umems[i].size == size && + virtq->umems[i].obj != NULL) { + /* Reuse registered memory. */ + memset(virtq->umems[i].buf, 0, size); + goto reuse; + } + if (virtq->umems[i].obj) + claim_zero(mlx5_glue->devx_umem_dereg (virtq->umems[i].obj)); - if (virtq->umems[i].buf) - rte_free(virtq->umems[i].buf); - virtq->umems[i].size = 0; - virtq->umems[i].obj = NULL; - virtq->umems[i].buf = NULL; - buf = rte_zmalloc(__func__, size, 4096); - if (buf == NULL) { - DRV_LOG(ERR, "Cannot allocate umem %d memory for virtq" + if (virtq->umems[i].buf) + rte_free(virtq->umems[i].buf); + virtq->umems[i].size = 0; + virtq->umems[i].obj = NULL; + virtq->umems[i].buf = NULL; + buf = rte_zmalloc(__func__, + size, 4096); + if (buf == NULL) { + DRV_LOG(ERR, "Cannot allocate umem %d memory for virtq" " %u.", i, index); - goto error; - } - obj = mlx5_glue->devx_umem_reg(priv->cdev->ctx, buf, size, - IBV_ACCESS_LOCAL_WRITE); - if (obj == NULL) { - DRV_LOG(ERR, "Failed to register umem %d for virtq %u.", + return -1; + } + obj = mlx5_glue->devx_umem_reg(priv->cdev->ctx, + buf, size, IBV_ACCESS_LOCAL_WRITE); + if (obj == NULL) { + DRV_LOG(ERR, "Failed to register umem %d for virtq %u.", i, index); - goto error; - } - virtq->umems[i].size = size; - virtq->umems[i].buf = buf; - virtq->umems[i].obj = obj; + rte_free(buf); + return -1; + } + virtq->umems[i].size = size; + virtq->umems[i].buf = buf; + virtq->umems[i].obj = obj; reuse: - attr.umems[i].id = virtq->umems[i].obj->umem_id; - attr.umems[i].offset = 0; - attr.umems[i].size = virtq->umems[i].size; + attr->umems[i].id = virtq->umems[i].obj->umem_id; + attr->umems[i].offset = 0; + attr->umems[i].size = virtq->umems[i].size; + } } - if (attr.type == MLX5_VIRTQ_TYPE_SPLIT) { + if (attr->q_type == MLX5_VIRTQ_TYPE_SPLIT) { gpa = mlx5_vdpa_hva_to_gpa(priv->vmem, - (uint64_t)(uintptr_t)vq.desc); + (uint64_t)(uintptr_t)vq->desc); if (!gpa) { DRV_LOG(ERR, "Failed to get descriptor ring GPA."); - goto error; + return -1; } - attr.desc_addr = gpa; + attr->desc_addr = gpa; gpa = mlx5_vdpa_hva_to_gpa(priv->vmem, - (uint64_t)(uintptr_t)vq.used); + (uint64_t)(uintptr_t)vq->used); if (!gpa) { DRV_LOG(ERR, "Failed to get GPA for used ring."); - goto error; + return -1; } - attr.used_addr = gpa; + attr->used_addr = gpa; gpa = mlx5_vdpa_hva_to_gpa(priv->vmem, - (uint64_t)(uintptr_t)vq.avail); + (uint64_t)(uintptr_t)vq->avail); if (!gpa) { DRV_LOG(ERR, "Failed to get GPA for available ring."); - goto error; + return -1; } - attr.available_addr = gpa; + attr->available_addr = gpa; } - ret = rte_vhost_get_vring_base(priv->vid, index, &last_avail_idx, - &last_used_idx); + ret = rte_vhost_get_vring_base(priv->vid, + index, &last_avail_idx, &last_used_idx); if (ret) { last_avail_idx = 0; last_used_idx = 0; @@ -345,25 +386,74 @@ reuse: "virtq %d.", priv->vid, last_avail_idx, last_used_idx, index); } - attr.hw_available_index = last_avail_idx; - attr.hw_used_index = last_used_idx; - attr.q_size = vq.size; - attr.mkey = priv->gpa_mkey_index; - attr.tis_id = priv->tiss[(index / 2) % priv->num_lag_ports]->id; - attr.queue_index = index; - attr.pd = priv->cdev->pdn; - attr.hw_latency_mode = priv->hw_latency_mode; - attr.hw_max_latency_us = priv->hw_max_latency_us; - attr.hw_max_pending_comp = priv->hw_max_pending_comp; - virtq->virtq = mlx5_devx_cmd_create_virtq(priv->cdev->ctx, &attr); + attr->hw_available_index = last_avail_idx; + attr->hw_used_index = last_used_idx; + attr->q_size = vq->size; + attr->mkey = priv->gpa_mkey_index; + attr->tis_id = priv->tiss[(index / 2) % priv->num_lag_ports]->id; + attr->queue_index = index; + attr->pd = priv->cdev->pdn; + attr->hw_latency_mode = priv->hw_latency_mode; + attr->hw_max_latency_us = priv->hw_max_latency_us; + attr->hw_max_pending_comp = priv->hw_max_pending_comp; + if (attr->hw_latency_mode || attr->hw_max_latency_us || + attr->hw_max_pending_comp) + attr->mod_fields_bitmap |= MLX5_VIRTQ_MODIFY_TYPE_QUEUE_PERIOD; + return 0; +} + +bool +mlx5_vdpa_is_modify_virtq_supported(struct mlx5_vdpa_priv *priv) +{ + return (priv->caps.vnet_modify_ext && + priv->caps.virtio_net_q_addr_modify && + priv->caps.virtio_q_index_modify) ? true : false; +} + +static int +mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv, int index) +{ + struct mlx5_vdpa_virtq *virtq = &priv->virtqs[index]; + struct rte_vhost_vring vq; + struct mlx5_devx_virtq_attr attr = {0}; + int ret; + uint16_t event_num = MLX5_EVENT_TYPE_OBJECT_CHANGE; + uint64_t cookie; + + ret = rte_vhost_get_vhost_vring(priv->vid, index, &vq); + if (ret) + return -1; + if (vq.size == 0) + return 0; virtq->priv = priv; - if (!virtq->virtq) + virtq->stopped = 0; + ret = mlx5_vdpa_virtq_sub_objs_prepare(priv, &attr, + &vq, index); + if (ret) { + DRV_LOG(ERR, "Failed to setup update virtq attr %d.", + index); goto error; - claim_zero(rte_vhost_enable_guest_notification(priv->vid, index, 1)); - if (mlx5_vdpa_virtq_modify(virtq, 1)) + } + if (!virtq->virtq) { + virtq->index = index; + virtq->vq_size = vq.size; + virtq->virtq = mlx5_devx_cmd_create_virtq(priv->cdev->ctx, + &attr); + if (!virtq->virtq) + goto error; + attr.mod_fields_bitmap = MLX5_VIRTQ_MODIFY_TYPE_STATE; + } + attr.state = MLX5_VIRTQ_STATE_RDY; + ret = mlx5_devx_cmd_modify_virtq(virtq->virtq, &attr); + if (ret) { + DRV_LOG(ERR, "Failed to modify virtq %d.", index); goto error; - virtq->priv = priv; + } + claim_zero(rte_vhost_enable_guest_notification(priv->vid, index, 1)); + virtq->configured = 1; + rte_spinlock_lock(&priv->db_lock); rte_write32(virtq->index, priv->virtq_db_addr); + rte_spinlock_unlock(&priv->db_lock); /* Setup doorbell mapping. */ virtq->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED); @@ -384,7 +474,7 @@ reuse: if (rte_intr_callback_register(virtq->intr_handle, mlx5_vdpa_virtq_kick_handler, virtq)) { - rte_intr_fd_set(virtq->intr_handle, -1); + (void)rte_intr_fd_set(virtq->intr_handle, -1); DRV_LOG(ERR, "Failed to register virtq %d interrupt.", index); goto error; @@ -480,6 +570,7 @@ mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv) uint32_t i; uint16_t nr_vring = rte_vhost_get_vring_num(priv->vid); int ret = rte_vhost_get_negotiated_features(priv->vid, &priv->features); + struct mlx5_vdpa_virtq *virtq; if (ret || mlx5_vdpa_features_validate(priv)) { DRV_LOG(ERR, "Failed to configure negotiated features."); @@ -499,9 +590,17 @@ mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv) return -1; } priv->nr_virtqs = nr_vring; - for (i = 0; i < nr_vring; i++) - if (priv->virtqs[i].enable && mlx5_vdpa_virtq_setup(priv, i)) - goto error; + for (i = 0; i < nr_vring; i++) { + virtq = &priv->virtqs[i]; + if (virtq->enable) { + pthread_mutex_lock(&virtq->virtq_lock); + if (mlx5_vdpa_virtq_setup(priv, i)) { + pthread_mutex_unlock(&virtq->virtq_lock); + goto error; + } + pthread_mutex_unlock(&virtq->virtq_lock); + } + } return 0; error: mlx5_vdpa_virtqs_release(priv); @@ -553,7 +652,7 @@ mlx5_vdpa_virtq_enable(struct mlx5_vdpa_priv *priv, int index, int enable) return 0; DRV_LOG(INFO, "Virtq %d was modified, recreate it.", index); } - if (virtq->virtq) { + if (virtq->configured) { virtq->enable = 0; if (is_virtq_recvq(virtq->index, priv->nr_virtqs)) { ret = mlx5_vdpa_steer_update(priv);