#include <net/if.h>
#include <sys/socket.h>
#include <sys/ioctl.h>
+#include <sys/mman.h>
#include <fcntl.h>
#include <netinet/in.h>
TAILQ_HEAD_INITIALIZER(priv_list);
static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
+static void mlx5_vdpa_dev_release(struct mlx5_vdpa_priv *priv);
+
static struct mlx5_vdpa_priv *
mlx5_vdpa_find_priv_resource_by_vdev(struct rte_vdpa_device *vdev)
{
DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
return -1;
}
- mlx5_vdpa_err_event_unset(priv);
mlx5_vdpa_cqe_event_unset(priv);
if (priv->state == MLX5_VDPA_STATE_CONFIGURED) {
ret |= mlx5_vdpa_lm_log(priv);
}
mlx5_vdpa_steer_unset(priv);
mlx5_vdpa_virtqs_release(priv);
- mlx5_vdpa_event_qp_global_release(priv);
mlx5_vdpa_mem_dereg(priv);
priv->state = MLX5_VDPA_STATE_PROBED;
priv->vid = 0;
if (mlx5_vdpa_mtu_set(priv))
DRV_LOG(WARNING, "MTU cannot be set on device %s.",
vdev->device->name);
- if (mlx5_vdpa_mem_register(priv) || mlx5_vdpa_err_event_setup(priv) ||
+ if (mlx5_vdpa_mem_register(priv) ||
mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) ||
mlx5_vdpa_cqe_event_setup(priv)) {
mlx5_vdpa_dev_close(vid);
DRV_LOG(DEBUG, "no traffic max is %u.", priv->no_traffic_max);
}
+static int
+mlx5_vdpa_create_dev_resources(struct mlx5_vdpa_priv *priv)
+{
+ struct mlx5_devx_tis_attr tis_attr = {0};
+ struct ibv_context *ctx = priv->cdev->ctx;
+ uint32_t i;
+ int retry;
+
+ for (retry = 0; retry < 7; retry++) {
+ priv->var = mlx5_glue->dv_alloc_var(ctx, 0);
+ if (priv->var != NULL)
+ break;
+ DRV_LOG(WARNING, "Failed to allocate VAR, retry %d.", retry);
+ /* Wait Qemu release VAR during vdpa restart, 0.1 sec based. */
+ usleep(100000U << retry);
+ }
+ if (!priv->var) {
+ DRV_LOG(ERR, "Failed to allocate VAR %u.", errno);
+ rte_errno = ENOMEM;
+ return -rte_errno;
+ }
+ /* Always map the entire page. */
+ priv->virtq_db_addr = mmap(NULL, priv->var->length, PROT_READ |
+ PROT_WRITE, MAP_SHARED, ctx->cmd_fd,
+ priv->var->mmap_off);
+ if (priv->virtq_db_addr == MAP_FAILED) {
+ DRV_LOG(ERR, "Failed to map doorbell page %u.", errno);
+ priv->virtq_db_addr = NULL;
+ rte_errno = errno;
+ return -rte_errno;
+ }
+ DRV_LOG(DEBUG, "VAR address of doorbell mapping is %p.",
+ priv->virtq_db_addr);
+ priv->td = mlx5_devx_cmd_create_td(ctx);
+ if (!priv->td) {
+ DRV_LOG(ERR, "Failed to create transport domain.");
+ rte_errno = errno;
+ return -rte_errno;
+ }
+ tis_attr.transport_domain = priv->td->id;
+ for (i = 0; i < priv->num_lag_ports; i++) {
+ /* 0 is auto affinity, non-zero value to propose port. */
+ tis_attr.lag_tx_port_affinity = i + 1;
+ priv->tiss[i] = mlx5_devx_cmd_create_tis(ctx, &tis_attr);
+ if (!priv->tiss[i]) {
+ DRV_LOG(ERR, "Failed to create TIS %u.", i);
+ return -rte_errno;
+ }
+ }
+ priv->null_mr = mlx5_glue->alloc_null_mr(priv->cdev->pd);
+ if (!priv->null_mr) {
+ DRV_LOG(ERR, "Failed to allocate null MR.");
+ rte_errno = errno;
+ return -rte_errno;
+ }
+ DRV_LOG(DEBUG, "Dump fill Mkey = %u.", priv->null_mr->lkey);
+#ifdef HAVE_MLX5DV_DR
+ priv->steer.domain = mlx5_glue->dr_create_domain(ctx,
+ MLX5DV_DR_DOMAIN_TYPE_NIC_RX);
+ if (!priv->steer.domain) {
+ DRV_LOG(ERR, "Failed to create Rx domain.");
+ rte_errno = errno;
+ return -rte_errno;
+ }
+#endif
+ priv->steer.tbl = mlx5_glue->dr_create_flow_tbl(priv->steer.domain, 0);
+ if (!priv->steer.tbl) {
+ DRV_LOG(ERR, "Failed to create table 0 with Rx domain.");
+ rte_errno = errno;
+ return -rte_errno;
+ }
+ if (mlx5_vdpa_err_event_setup(priv) != 0)
+ return -rte_errno;
+ if (mlx5_vdpa_event_qp_global_prepare(priv))
+ return -rte_errno;
+ return 0;
+}
+
static int
mlx5_vdpa_dev_probe(struct mlx5_common_device *cdev,
struct mlx5_kvargs_ctrl *mkvlist)
{
struct mlx5_vdpa_priv *priv = NULL;
struct mlx5_hca_attr *attr = &cdev->config.hca_attr;
- int retry;
if (!attr->vdpa.valid || !attr->vdpa.max_num_virtio_queues) {
DRV_LOG(ERR, "Not enough capabilities to support vdpa, maybe "
priv->num_lag_ports = attr->num_lag_ports;
if (attr->num_lag_ports == 0)
priv->num_lag_ports = 1;
+ pthread_mutex_init(&priv->vq_config_lock, NULL);
priv->cdev = cdev;
- for (retry = 0; retry < 7; retry++) {
- priv->var = mlx5_glue->dv_alloc_var(priv->cdev->ctx, 0);
- if (priv->var != NULL)
- break;
- DRV_LOG(WARNING, "Failed to allocate VAR, retry %d.\n", retry);
- /* Wait Qemu release VAR during vdpa restart, 0.1 sec based. */
- usleep(100000U << retry);
- }
- if (!priv->var) {
- DRV_LOG(ERR, "Failed to allocate VAR %u.", errno);
+ if (mlx5_vdpa_create_dev_resources(priv))
goto error;
- }
- priv->err_intr_handle =
- rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
- if (priv->err_intr_handle == NULL) {
- DRV_LOG(ERR, "Fail to allocate intr_handle");
- goto error;
- }
priv->vdev = rte_vdpa_register_device(cdev->dev, &mlx5_vdpa_ops);
if (priv->vdev == NULL) {
DRV_LOG(ERR, "Failed to register vDPA device.");
}
mlx5_vdpa_config_get(mkvlist, priv);
SLIST_INIT(&priv->mr_list);
- pthread_mutex_init(&priv->vq_config_lock, NULL);
pthread_mutex_lock(&priv_list_lock);
TAILQ_INSERT_TAIL(&priv_list, priv, next);
pthread_mutex_unlock(&priv_list_lock);
return 0;
-
error:
- if (priv) {
- if (priv->var)
- mlx5_glue->dv_free_var(priv->var);
- rte_intr_instance_free(priv->err_intr_handle);
- rte_free(priv);
- }
+ if (priv)
+ mlx5_vdpa_dev_release(priv);
return -rte_errno;
}
if (found)
TAILQ_REMOVE(&priv_list, priv, next);
pthread_mutex_unlock(&priv_list_lock);
- if (found) {
- if (priv->state == MLX5_VDPA_STATE_CONFIGURED)
- mlx5_vdpa_dev_close(priv->vid);
- if (priv->var) {
- mlx5_glue->dv_free_var(priv->var);
- priv->var = NULL;
- }
- if (priv->vdev)
- rte_vdpa_unregister_device(priv->vdev);
- pthread_mutex_destroy(&priv->vq_config_lock);
- rte_intr_instance_free(priv->err_intr_handle);
- rte_free(priv);
- }
+ if (found)
+ mlx5_vdpa_dev_release(priv);
return 0;
}
+static void
+mlx5_vdpa_release_dev_resources(struct mlx5_vdpa_priv *priv)
+{
+ uint32_t i;
+
+ mlx5_vdpa_event_qp_global_release(priv);
+ mlx5_vdpa_err_event_unset(priv);
+ if (priv->steer.tbl)
+ claim_zero(mlx5_glue->dr_destroy_flow_tbl(priv->steer.tbl));
+ if (priv->steer.domain)
+ claim_zero(mlx5_glue->dr_destroy_domain(priv->steer.domain));
+ if (priv->null_mr)
+ claim_zero(mlx5_glue->dereg_mr(priv->null_mr));
+ for (i = 0; i < priv->num_lag_ports; i++) {
+ if (priv->tiss[i])
+ claim_zero(mlx5_devx_cmd_destroy(priv->tiss[i]));
+ }
+ if (priv->td)
+ claim_zero(mlx5_devx_cmd_destroy(priv->td));
+ if (priv->virtq_db_addr)
+ claim_zero(munmap(priv->virtq_db_addr, priv->var->length));
+ if (priv->var)
+ mlx5_glue->dv_free_var(priv->var);
+}
+
+static void
+mlx5_vdpa_dev_release(struct mlx5_vdpa_priv *priv)
+{
+ if (priv->state == MLX5_VDPA_STATE_CONFIGURED)
+ mlx5_vdpa_dev_close(priv->vid);
+ mlx5_vdpa_release_dev_resources(priv);
+ if (priv->vdev)
+ rte_vdpa_unregister_device(priv->vdev);
+ pthread_mutex_destroy(&priv->vq_config_lock);
+ rte_free(priv);
+}
+
static const struct rte_pci_id mlx5_vdpa_pci_id_map[] = {
{
RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
}
/* Prepare all the global resources for all the event objects.*/
-static int
+int
mlx5_vdpa_event_qp_global_prepare(struct mlx5_vdpa_priv *priv)
{
- if (priv->eventc)
- return 0;
priv->eventc = mlx5_os_devx_create_event_channel(priv->cdev->ctx,
MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA);
if (!priv->eventc) {
flags = fcntl(priv->err_chnl->fd, F_GETFL);
ret = fcntl(priv->err_chnl->fd, F_SETFL, flags | O_NONBLOCK);
if (ret) {
+ rte_errno = errno;
DRV_LOG(ERR, "Failed to change device event channel FD.");
goto error;
}
-
+ priv->err_intr_handle =
+ rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
+ if (priv->err_intr_handle == NULL) {
+ DRV_LOG(ERR, "Fail to allocate intr_handle");
+ goto error;
+ }
if (rte_intr_fd_set(priv->err_intr_handle, priv->err_chnl->fd))
goto error;
if (rte_intr_type_set(priv->err_intr_handle, RTE_INTR_HANDLE_EXT))
goto error;
- if (rte_intr_callback_register(priv->err_intr_handle,
- mlx5_vdpa_err_interrupt_handler,
- priv)) {
+ ret = rte_intr_callback_register(priv->err_intr_handle,
+ mlx5_vdpa_err_interrupt_handler,
+ priv);
+ if (ret != 0) {
rte_intr_fd_set(priv->err_intr_handle, 0);
DRV_LOG(ERR, "Failed to register error interrupt for device %d.",
priv->vid);
+ rte_errno = -ret;
goto error;
} else {
DRV_LOG(DEBUG, "Registered error interrupt for device%d.",
mlx5_glue->devx_destroy_event_channel(priv->err_chnl);
priv->err_chnl = NULL;
}
+ rte_intr_instance_free(priv->err_intr_handle);
}
int
uint16_t log_desc_n = rte_log2_u32(desc_n);
uint32_t ret;
- if (mlx5_vdpa_event_qp_global_prepare(priv))
- return -1;
if (mlx5_vdpa_cq_create(priv, log_desc_n, callfd, &eqp->cq))
return -1;
attr.pd = priv->cdev->pdn;
*/
#include <string.h>
#include <unistd.h>
-#include <sys/mman.h>
#include <sys/eventfd.h>
#include <rte_malloc.h>
if (virtq->counters)
claim_zero(mlx5_devx_cmd_destroy(virtq->counters));
}
- for (i = 0; i < priv->num_lag_ports; i++) {
- if (priv->tiss[i]) {
- claim_zero(mlx5_devx_cmd_destroy(priv->tiss[i]));
- priv->tiss[i] = NULL;
- }
- }
- if (priv->td) {
- claim_zero(mlx5_devx_cmd_destroy(priv->td));
- priv->td = NULL;
- }
- if (priv->virtq_db_addr) {
- claim_zero(munmap(priv->virtq_db_addr, priv->var->length));
- priv->virtq_db_addr = NULL;
- }
priv->features = 0;
memset(priv->virtqs, 0, sizeof(*virtq) * priv->nr_virtqs);
priv->nr_virtqs = 0;
int
mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv)
{
- struct mlx5_devx_tis_attr tis_attr = {0};
- struct ibv_context *ctx = priv->cdev->ctx;
uint32_t i;
uint16_t nr_vring = rte_vhost_get_vring_num(priv->vid);
int ret = rte_vhost_get_negotiated_features(priv->vid, &priv->features);
(int)nr_vring);
return -1;
}
- /* Always map the entire page. */
- priv->virtq_db_addr = mmap(NULL, priv->var->length, PROT_READ |
- PROT_WRITE, MAP_SHARED, ctx->cmd_fd,
- priv->var->mmap_off);
- if (priv->virtq_db_addr == MAP_FAILED) {
- DRV_LOG(ERR, "Failed to map doorbell page %u.", errno);
- priv->virtq_db_addr = NULL;
- goto error;
- } else {
- DRV_LOG(DEBUG, "VAR address of doorbell mapping is %p.",
- priv->virtq_db_addr);
- }
- priv->td = mlx5_devx_cmd_create_td(ctx);
- if (!priv->td) {
- DRV_LOG(ERR, "Failed to create transport domain.");
- return -rte_errno;
- }
- tis_attr.transport_domain = priv->td->id;
- for (i = 0; i < priv->num_lag_ports; i++) {
- /* 0 is auto affinity, non-zero value to propose port. */
- tis_attr.lag_tx_port_affinity = i + 1;
- priv->tiss[i] = mlx5_devx_cmd_create_tis(ctx, &tis_attr);
- if (!priv->tiss[i]) {
- DRV_LOG(ERR, "Failed to create TIS %u.", i);
- goto error;
- }
- }
priv->nr_virtqs = nr_vring;
for (i = 0; i < nr_vring; i++)
if (priv->virtqs[i].enable && mlx5_vdpa_virtq_setup(priv, i))
git.droids-corp.org / dpdk.git / commitdiff