+static int
+mlx5_vdpa_set_vring_state(int vid, int vring, int state)
+{
+ struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
+ struct mlx5_vdpa_priv *priv =
+ mlx5_vdpa_find_priv_resource_by_vdev(vdev);
+
+ if (priv == NULL) {
+ DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
+ return -EINVAL;
+ }
+ if (vring >= (int)priv->caps.max_num_virtio_queues * 2) {
+ DRV_LOG(ERR, "Too big vring id: %d.", vring);
+ return -E2BIG;
+ }
+ return mlx5_vdpa_virtq_enable(priv, vring, state);
+}
+
+static int
+mlx5_vdpa_direct_db_prepare(struct mlx5_vdpa_priv *priv)
+{
+ int ret;
+
+ if (priv->direct_notifier) {
+ ret = rte_vhost_host_notifier_ctrl(priv->vid, false);
+ if (ret != 0) {
+ DRV_LOG(INFO, "Direct HW notifier FD cannot be "
+ "destroyed for device %d: %d.", priv->vid, ret);
+ return -1;
+ }
+ priv->direct_notifier = 0;
+ }
+ ret = rte_vhost_host_notifier_ctrl(priv->vid, true);
+ if (ret != 0)
+ DRV_LOG(INFO, "Direct HW notifier FD cannot be configured for"
+ " device %d: %d.", priv->vid, ret);
+ else
+ priv->direct_notifier = 1;
+ return 0;
+}
+
+static int
+mlx5_vdpa_features_set(int vid)
+{
+ struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
+ struct mlx5_vdpa_priv *priv =
+ mlx5_vdpa_find_priv_resource_by_vdev(vdev);
+ uint64_t log_base, log_size;
+ uint64_t features;
+ int ret;
+
+ if (priv == NULL) {
+ DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
+ return -EINVAL;
+ }
+ ret = rte_vhost_get_negotiated_features(vid, &features);
+ if (ret) {
+ DRV_LOG(ERR, "Failed to get negotiated features.");
+ return ret;
+ }
+ if (RTE_VHOST_NEED_LOG(features)) {
+ ret = rte_vhost_get_log_base(vid, &log_base, &log_size);
+ if (ret) {
+ DRV_LOG(ERR, "Failed to get log base.");
+ return ret;
+ }
+ ret = mlx5_vdpa_dirty_bitmap_set(priv, log_base, log_size);
+ if (ret) {
+ DRV_LOG(ERR, "Failed to set dirty bitmap.");
+ return ret;
+ }
+ DRV_LOG(INFO, "mlx5 vdpa: enabling dirty logging...");
+ ret = mlx5_vdpa_logging_enable(priv, 1);
+ if (ret) {
+ DRV_LOG(ERR, "Failed t enable dirty logging.");
+ return ret;
+ }
+ }
+ return 0;
+}
+
+static int
+mlx5_vdpa_pd_create(struct mlx5_vdpa_priv *priv)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ priv->pd = mlx5_glue->alloc_pd(priv->ctx);
+ if (priv->pd == NULL) {
+ DRV_LOG(ERR, "Failed to allocate PD.");
+ return errno ? -errno : -ENOMEM;
+ }
+ struct mlx5dv_obj obj;
+ struct mlx5dv_pd pd_info;
+ int ret = 0;
+
+ obj.pd.in = priv->pd;
+ obj.pd.out = &pd_info;
+ ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+ if (ret) {
+ DRV_LOG(ERR, "Fail to get PD object info.");
+ mlx5_glue->dealloc_pd(priv->pd);
+ priv->pd = NULL;
+ return -errno;
+ }
+ priv->pdn = pd_info.pdn;
+ return 0;
+#else
+ (void)priv;
+ DRV_LOG(ERR, "Cannot get pdn - no DV support.");
+ return -ENOTSUP;
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+}
+
+static int
+mlx5_vdpa_mtu_set(struct mlx5_vdpa_priv *priv)
+{
+ struct ifreq request;
+ uint16_t vhost_mtu = 0;
+ uint16_t kern_mtu = 0;
+ int ret = rte_vhost_get_mtu(priv->vid, &vhost_mtu);
+ int sock;
+ int retries = MLX5_VDPA_MAX_RETRIES;
+
+ if (ret) {
+ DRV_LOG(DEBUG, "Cannot get vhost MTU - %d.", ret);
+ return ret;
+ }
+ if (!vhost_mtu) {
+ DRV_LOG(DEBUG, "Vhost MTU is 0.");
+ return ret;
+ }
+ ret = mlx5_get_ifname_sysfs(priv->ctx->device->ibdev_path,
+ request.ifr_name);
+ if (ret) {
+ DRV_LOG(DEBUG, "Cannot get kernel IF name - %d.", ret);
+ return ret;
+ }
+ sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
+ if (sock == -1) {
+ DRV_LOG(DEBUG, "Cannot open IF socket.");
+ return sock;
+ }
+ while (retries--) {
+ ret = ioctl(sock, SIOCGIFMTU, &request);
+ if (ret == -1)
+ break;
+ kern_mtu = request.ifr_mtu;
+ DRV_LOG(DEBUG, "MTU: current %d requested %d.", (int)kern_mtu,
+ (int)vhost_mtu);
+ if (kern_mtu == vhost_mtu)
+ break;
+ request.ifr_mtu = vhost_mtu;
+ ret = ioctl(sock, SIOCSIFMTU, &request);
+ if (ret == -1)
+ break;
+ request.ifr_mtu = 0;
+ usleep(MLX5_VDPA_USEC);
+ }
+ close(sock);
+ return kern_mtu == vhost_mtu ? 0 : -1;
+}
+
+static int
+mlx5_vdpa_dev_close(int vid)
+{
+ struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
+ struct mlx5_vdpa_priv *priv =
+ mlx5_vdpa_find_priv_resource_by_vdev(vdev);
+ int ret = 0;
+
+ if (priv == NULL) {
+ DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
+ return -1;
+ }
+ if (priv->configured)
+ ret |= mlx5_vdpa_lm_log(priv);
+ mlx5_vdpa_cqe_event_unset(priv);
+ mlx5_vdpa_steer_unset(priv);
+ mlx5_vdpa_virtqs_release(priv);
+ mlx5_vdpa_event_qp_global_release(priv);
+ mlx5_vdpa_mem_dereg(priv);
+ if (priv->pd) {
+ claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+ priv->pd = NULL;
+ }
+ priv->configured = 0;
+ priv->vid = 0;
+ DRV_LOG(INFO, "vDPA device %d was closed.", vid);
+ return ret;
+}
+
+static int
+mlx5_vdpa_dev_config(int vid)
+{
+ struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
+ struct mlx5_vdpa_priv *priv =
+ mlx5_vdpa_find_priv_resource_by_vdev(vdev);
+
+ if (priv == NULL) {
+ DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
+ return -EINVAL;
+ }
+ if (priv->configured && mlx5_vdpa_dev_close(vid)) {
+ DRV_LOG(ERR, "Failed to reconfigure vid %d.", vid);
+ return -1;
+ }
+ priv->vid = vid;
+ if (mlx5_vdpa_mtu_set(priv))
+ DRV_LOG(WARNING, "MTU cannot be set on device %s.",
+ vdev->device->name);
+ if (mlx5_vdpa_pd_create(priv) || mlx5_vdpa_mem_register(priv) ||
+ mlx5_vdpa_direct_db_prepare(priv) ||
+ mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) ||
+ mlx5_vdpa_cqe_event_setup(priv)) {
+ mlx5_vdpa_dev_close(vid);
+ return -1;
+ }
+ priv->configured = 1;
+ DRV_LOG(INFO, "vDPA device %d was configured.", vid);
+ return 0;
+}
+
+static int
+mlx5_vdpa_get_device_fd(int vid)
+{
+ struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
+ struct mlx5_vdpa_priv *priv =
+ mlx5_vdpa_find_priv_resource_by_vdev(vdev);
+
+ if (priv == NULL) {
+ DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
+ return -EINVAL;
+ }
+ return priv->ctx->cmd_fd;
+}
+
+static int
+mlx5_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
+{
+ struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
+ struct mlx5_vdpa_priv *priv =
+ mlx5_vdpa_find_priv_resource_by_vdev(vdev);
+
+ RTE_SET_USED(qid);
+ if (priv == NULL) {
+ DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
+ return -EINVAL;
+ }
+ if (!priv->var) {
+ DRV_LOG(ERR, "VAR was not created for device %s, is the device"
+ " configured?.", vdev->device->name);
+ return -EINVAL;
+ }
+ *offset = priv->var->mmap_off;
+ *size = priv->var->length;
+ return 0;
+}
+
+static int
+mlx5_vdpa_get_stats_names(struct rte_vdpa_device *vdev,
+ struct rte_vdpa_stat_name *stats_names,
+ unsigned int size)
+{
+ static const char *mlx5_vdpa_stats_names[MLX5_VDPA_STATS_MAX] = {
+ "received_descriptors",
+ "completed_descriptors",
+ "bad descriptor errors",
+ "exceed max chain",
+ "invalid buffer",
+ "completion errors",
+ };
+ struct mlx5_vdpa_priv *priv =
+ mlx5_vdpa_find_priv_resource_by_vdev(vdev);
+ unsigned int i;
+
+ if (priv == NULL) {
+ DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
+ return -ENODEV;
+ }
+ if (!stats_names)
+ return MLX5_VDPA_STATS_MAX;
+ size = RTE_MIN(size, (unsigned int)MLX5_VDPA_STATS_MAX);
+ for (i = 0; i < size; ++i)
+ strlcpy(stats_names[i].name, mlx5_vdpa_stats_names[i],
+ RTE_VDPA_STATS_NAME_SIZE);
+ return size;
+}
+
+static int
+mlx5_vdpa_get_stats(struct rte_vdpa_device *vdev, int qid,
+ struct rte_vdpa_stat *stats, unsigned int n)
+{
+ struct mlx5_vdpa_priv *priv =
+ mlx5_vdpa_find_priv_resource_by_vdev(vdev);
+
+ if (priv == NULL) {
+ DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
+ return -ENODEV;
+ }
+ if (!priv->configured) {
+ DRV_LOG(ERR, "Device %s was not configured.",
+ vdev->device->name);
+ return -ENODATA;
+ }
+ if (qid >= (int)priv->nr_virtqs) {
+ DRV_LOG(ERR, "Too big vring id: %d for device %s.", qid,
+ vdev->device->name);
+ return -E2BIG;
+ }
+ if (!priv->caps.queue_counters_valid) {
+ DRV_LOG(ERR, "Virtq statistics is not supported for device %s.",
+ vdev->device->name);
+ return -ENOTSUP;
+ }
+ return mlx5_vdpa_virtq_stats_get(priv, qid, stats, n);
+}
+
+static int
+mlx5_vdpa_reset_stats(struct rte_vdpa_device *vdev, int qid)
+{
+ struct mlx5_vdpa_priv *priv =
+ mlx5_vdpa_find_priv_resource_by_vdev(vdev);
+
+ if (priv == NULL) {
+ DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
+ return -ENODEV;
+ }
+ if (!priv->configured) {
+ DRV_LOG(ERR, "Device %s was not configured.",
+ vdev->device->name);
+ return -ENODATA;
+ }
+ if (qid >= (int)priv->nr_virtqs) {
+ DRV_LOG(ERR, "Too big vring id: %d for device %s.", qid,
+ vdev->device->name);
+ return -E2BIG;
+ }
+ if (!priv->caps.queue_counters_valid) {
+ DRV_LOG(ERR, "Virtq statistics is not supported for device %s.",
+ vdev->device->name);
+ return -ENOTSUP;
+ }
+ return mlx5_vdpa_virtq_stats_reset(priv, qid);
+}
+