drivers/vdpa/mlx5/mlx5_vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <unistd.h>
   5 #include <net/if.h>
   6 #include <sys/socket.h>
   7 #include <sys/ioctl.h>
   8 #include <fcntl.h>
   9 #include <netinet/in.h>
  10
  11 #include <rte_malloc.h>
  12 #include <rte_log.h>
  13 #include <rte_errno.h>
  14 #include <rte_string_fns.h>
  15 #include <rte_bus_pci.h>
  16
  17 #include <mlx5_glue.h>
  18 #include <mlx5_common.h>
  19 #include <mlx5_common_defs.h>
  20 #include <mlx5_devx_cmds.h>
  21 #include <mlx5_prm.h>
  22 #include <mlx5_nl.h>
  23
  24 #include "mlx5_vdpa_utils.h"
  25 #include "mlx5_vdpa.h"
  26
  27 #define MLX5_VDPA_DRIVER_NAME vdpa_mlx5
  28
  29 #define MLX5_VDPA_DEFAULT_FEATURES ((1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
  30                             (1ULL << VIRTIO_F_ANY_LAYOUT) | \
  31                             (1ULL << VIRTIO_NET_F_MQ) | \
  32                             (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \
  33                             (1ULL << VIRTIO_F_ORDER_PLATFORM) | \
  34                             (1ULL << VHOST_F_LOG_ALL) | \
  35                             (1ULL << VIRTIO_NET_F_MTU))
  36
  37 #define MLX5_VDPA_PROTOCOL_FEATURES \
  38                             ((1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
  39                              (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
  40                              (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
  41                              (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) | \
  42                              (1ULL << VHOST_USER_PROTOCOL_F_MQ) | \
  43                              (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU) | \
  44                              (1ULL << VHOST_USER_PROTOCOL_F_STATUS))
  45
  46 #define MLX5_VDPA_DEFAULT_NO_TRAFFIC_MAX 16LLU
  47
  48 TAILQ_HEAD(mlx5_vdpa_privs, mlx5_vdpa_priv) priv_list =
  49                                               TAILQ_HEAD_INITIALIZER(priv_list);
  50 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
  51
  52 static struct mlx5_vdpa_priv *
  53 mlx5_vdpa_find_priv_resource_by_vdev(struct rte_vdpa_device *vdev)
  54 {
  55         struct mlx5_vdpa_priv *priv;
  56         int found = 0;
  57
  58         pthread_mutex_lock(&priv_list_lock);
  59         TAILQ_FOREACH(priv, &priv_list, next) {
  60                 if (vdev == priv->vdev) {
  61                         found = 1;
  62                         break;
  63                 }
  64         }
  65         pthread_mutex_unlock(&priv_list_lock);
  66         if (!found) {
  67                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
  68                 rte_errno = EINVAL;
  69                 return NULL;
  70         }
  71         return priv;
  72 }
  73
  74 static int
  75 mlx5_vdpa_get_queue_num(struct rte_vdpa_device *vdev, uint32_t *queue_num)
  76 {
  77         struct mlx5_vdpa_priv *priv =
  78                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
  79
  80         if (priv == NULL) {
  81                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
  82                 return -1;
  83         }
  84         *queue_num = priv->caps.max_num_virtio_queues;
  85         return 0;
  86 }
  87
  88 static int
  89 mlx5_vdpa_get_vdpa_features(struct rte_vdpa_device *vdev, uint64_t *features)
  90 {
  91         struct mlx5_vdpa_priv *priv =
  92                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
  93
  94         if (priv == NULL) {
  95                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
  96                 return -1;
  97         }
  98         *features = MLX5_VDPA_DEFAULT_FEATURES;
  99         if (priv->caps.virtio_queue_type & (1 << MLX5_VIRTQ_TYPE_PACKED))
 100                 *features |= (1ULL << VIRTIO_F_RING_PACKED);
 101         if (priv->caps.tso_ipv4)
 102                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO4);
 103         if (priv->caps.tso_ipv6)
 104                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO6);
 105         if (priv->caps.tx_csum)
 106                 *features |= (1ULL << VIRTIO_NET_F_CSUM);
 107         if (priv->caps.rx_csum)
 108                 *features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
 109         if (priv->caps.virtio_version_1_0)
 110                 *features |= (1ULL << VIRTIO_F_VERSION_1);
 111         return 0;
 112 }
 113
 114 static int
 115 mlx5_vdpa_get_protocol_features(struct rte_vdpa_device *vdev,
 116                 uint64_t *features)
 117 {
 118         struct mlx5_vdpa_priv *priv =
 119                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 120
 121         if (priv == NULL) {
 122                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 123                 return -1;
 124         }
 125         *features = MLX5_VDPA_PROTOCOL_FEATURES;
 126         return 0;
 127 }
 128
 129 static int
 130 mlx5_vdpa_set_vring_state(int vid, int vring, int state)
 131 {
 132         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 133         struct mlx5_vdpa_priv *priv =
 134                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 135         int ret;
 136
 137         if (priv == NULL) {
 138                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 139                 return -EINVAL;
 140         }
 141         if (vring >= (int)priv->caps.max_num_virtio_queues * 2) {
 142                 DRV_LOG(ERR, "Too big vring id: %d.", vring);
 143                 return -E2BIG;
 144         }
 145         pthread_mutex_lock(&priv->vq_config_lock);
 146         ret = mlx5_vdpa_virtq_enable(priv, vring, state);
 147         pthread_mutex_unlock(&priv->vq_config_lock);
 148         return ret;
 149 }
 150
 151 static int
 152 mlx5_vdpa_features_set(int vid)
 153 {
 154         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 155         struct mlx5_vdpa_priv *priv =
 156                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 157         uint64_t log_base, log_size;
 158         uint64_t features;
 159         int ret;
 160
 161         if (priv == NULL) {
 162                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 163                 return -EINVAL;
 164         }
 165         ret = rte_vhost_get_negotiated_features(vid, &features);
 166         if (ret) {
 167                 DRV_LOG(ERR, "Failed to get negotiated features.");
 168                 return ret;
 169         }
 170         if (RTE_VHOST_NEED_LOG(features)) {
 171                 ret = rte_vhost_get_log_base(vid, &log_base, &log_size);
 172                 if (ret) {
 173                         DRV_LOG(ERR, "Failed to get log base.");
 174                         return ret;
 175                 }
 176                 ret = mlx5_vdpa_dirty_bitmap_set(priv, log_base, log_size);
 177                 if (ret) {
 178                         DRV_LOG(ERR, "Failed to set dirty bitmap.");
 179                         return ret;
 180                 }
 181                 DRV_LOG(INFO, "mlx5 vdpa: enabling dirty logging...");
 182                 ret = mlx5_vdpa_logging_enable(priv, 1);
 183                 if (ret) {
 184                         DRV_LOG(ERR, "Failed t enable dirty logging.");
 185                         return ret;
 186                 }
 187         }
 188         return 0;
 189 }
 190
 191 static int
 192 mlx5_vdpa_mtu_set(struct mlx5_vdpa_priv *priv)
 193 {
 194         struct ifreq request;
 195         uint16_t vhost_mtu = 0;
 196         uint16_t kern_mtu = 0;
 197         int ret = rte_vhost_get_mtu(priv->vid, &vhost_mtu);
 198         int sock;
 199         int retries = MLX5_VDPA_MAX_RETRIES;
 200
 201         if (ret) {
 202                 DRV_LOG(DEBUG, "Cannot get vhost MTU - %d.", ret);
 203                 return ret;
 204         }
 205         if (!vhost_mtu) {
 206                 DRV_LOG(DEBUG, "Vhost MTU is 0.");
 207                 return ret;
 208         }
 209         ret = mlx5_get_ifname_sysfs
 210                                 (mlx5_os_get_ctx_device_name(priv->cdev->ctx),
 211                                  request.ifr_name);
 212         if (ret) {
 213                 DRV_LOG(DEBUG, "Cannot get kernel IF name - %d.", ret);
 214                 return ret;
 215         }
 216         sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
 217         if (sock == -1) {
 218                 DRV_LOG(DEBUG, "Cannot open IF socket.");
 219                 return sock;
 220         }
 221         while (retries--) {
 222                 ret = ioctl(sock, SIOCGIFMTU, &request);
 223                 if (ret == -1)
 224                         break;
 225                 kern_mtu = request.ifr_mtu;
 226                 DRV_LOG(DEBUG, "MTU: current %d requested %d.", (int)kern_mtu,
 227                         (int)vhost_mtu);
 228                 if (kern_mtu == vhost_mtu)
 229                         break;
 230                 request.ifr_mtu = vhost_mtu;
 231                 ret = ioctl(sock, SIOCSIFMTU, &request);
 232                 if (ret == -1)
 233                         break;
 234                 request.ifr_mtu = 0;
 235                 usleep(MLX5_VDPA_USEC);
 236         }
 237         close(sock);
 238         return kern_mtu == vhost_mtu ? 0 : -1;
 239 }
 240
 241 static int
 242 mlx5_vdpa_dev_close(int vid)
 243 {
 244         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 245         struct mlx5_vdpa_priv *priv =
 246                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 247         int ret = 0;
 248
 249         if (priv == NULL) {
 250                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 251                 return -1;
 252         }
 253         mlx5_vdpa_err_event_unset(priv);
 254         mlx5_vdpa_cqe_event_unset(priv);
 255         if (priv->configured)
 256                 ret |= mlx5_vdpa_lm_log(priv);
 257         mlx5_vdpa_steer_unset(priv);
 258         mlx5_vdpa_virtqs_release(priv);
 259         mlx5_vdpa_event_qp_global_release(priv);
 260         mlx5_vdpa_mem_dereg(priv);
 261         priv->configured = 0;
 262         priv->vid = 0;
 263         /* The mutex may stay locked after event thread cancel - initiate it. */
 264         pthread_mutex_init(&priv->vq_config_lock, NULL);
 265         DRV_LOG(INFO, "vDPA device %d was closed.", vid);
 266         return ret;
 267 }
 268
 269 static int
 270 mlx5_vdpa_dev_config(int vid)
 271 {
 272         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 273         struct mlx5_vdpa_priv *priv =
 274                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 275
 276         if (priv == NULL) {
 277                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 278                 return -EINVAL;
 279         }
 280         if (priv->configured && mlx5_vdpa_dev_close(vid)) {
 281                 DRV_LOG(ERR, "Failed to reconfigure vid %d.", vid);
 282                 return -1;
 283         }
 284         priv->vid = vid;
 285         if (mlx5_vdpa_mtu_set(priv))
 286                 DRV_LOG(WARNING, "MTU cannot be set on device %s.",
 287                                 vdev->device->name);
 288         if (mlx5_vdpa_mem_register(priv) || mlx5_vdpa_err_event_setup(priv) ||
 289             mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) ||
 290             mlx5_vdpa_cqe_event_setup(priv)) {
 291                 mlx5_vdpa_dev_close(vid);
 292                 return -1;
 293         }
 294         priv->configured = 1;
 295         DRV_LOG(INFO, "vDPA device %d was configured.", vid);
 296         return 0;
 297 }
 298
 299 static int
 300 mlx5_vdpa_get_device_fd(int vid)
 301 {
 302         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 303         struct mlx5_vdpa_priv *priv =
 304                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 305
 306         if (priv == NULL) {
 307                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 308                 return -EINVAL;
 309         }
 310         return ((struct ibv_context *)priv->cdev->ctx)->cmd_fd;
 311 }
 312
 313 static int
 314 mlx5_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
 315 {
 316         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 317         struct mlx5_vdpa_priv *priv =
 318                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 319
 320         RTE_SET_USED(qid);
 321         if (priv == NULL) {
 322                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 323                 return -EINVAL;
 324         }
 325         if (!priv->var) {
 326                 DRV_LOG(ERR, "VAR was not created for device %s, is the device"
 327                         " configured?.", vdev->device->name);
 328                 return -EINVAL;
 329         }
 330         *offset = priv->var->mmap_off;
 331         *size = priv->var->length;
 332         return 0;
 333 }
 334
 335 static int
 336 mlx5_vdpa_get_stats_names(struct rte_vdpa_device *vdev,
 337                 struct rte_vdpa_stat_name *stats_names,
 338                 unsigned int size)
 339 {
 340         static const char *mlx5_vdpa_stats_names[MLX5_VDPA_STATS_MAX] = {
 341                 "received_descriptors",
 342                 "completed_descriptors",
 343                 "bad descriptor errors",
 344                 "exceed max chain",
 345                 "invalid buffer",
 346                 "completion errors",
 347         };
 348         struct mlx5_vdpa_priv *priv =
 349                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 350         unsigned int i;
 351
 352         if (priv == NULL) {
 353                 DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
 354                 return -ENODEV;
 355         }
 356         if (!stats_names)
 357                 return MLX5_VDPA_STATS_MAX;
 358         size = RTE_MIN(size, (unsigned int)MLX5_VDPA_STATS_MAX);
 359         for (i = 0; i < size; ++i)
 360                 strlcpy(stats_names[i].name, mlx5_vdpa_stats_names[i],
 361                         RTE_VDPA_STATS_NAME_SIZE);
 362         return size;
 363 }
 364
 365 static int
 366 mlx5_vdpa_get_stats(struct rte_vdpa_device *vdev, int qid,
 367                 struct rte_vdpa_stat *stats, unsigned int n)
 368 {
 369         struct mlx5_vdpa_priv *priv =
 370                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 371
 372         if (priv == NULL) {
 373                 DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
 374                 return -ENODEV;
 375         }
 376         if (!priv->configured) {
 377                 DRV_LOG(ERR, "Device %s was not configured.",
 378                                 vdev->device->name);
 379                 return -ENODATA;
 380         }
 381         if (qid >= (int)priv->nr_virtqs) {
 382                 DRV_LOG(ERR, "Too big vring id: %d for device %s.", qid,
 383                                 vdev->device->name);
 384                 return -E2BIG;
 385         }
 386         if (!priv->caps.queue_counters_valid) {
 387                 DRV_LOG(ERR, "Virtq statistics is not supported for device %s.",
 388                         vdev->device->name);
 389                 return -ENOTSUP;
 390         }
 391         return mlx5_vdpa_virtq_stats_get(priv, qid, stats, n);
 392 }
 393
 394 static int
 395 mlx5_vdpa_reset_stats(struct rte_vdpa_device *vdev, int qid)
 396 {
 397         struct mlx5_vdpa_priv *priv =
 398                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 399
 400         if (priv == NULL) {
 401                 DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
 402                 return -ENODEV;
 403         }
 404         if (!priv->configured) {
 405                 DRV_LOG(ERR, "Device %s was not configured.",
 406                                 vdev->device->name);
 407                 return -ENODATA;
 408         }
 409         if (qid >= (int)priv->nr_virtqs) {
 410                 DRV_LOG(ERR, "Too big vring id: %d for device %s.", qid,
 411                                 vdev->device->name);
 412                 return -E2BIG;
 413         }
 414         if (!priv->caps.queue_counters_valid) {
 415                 DRV_LOG(ERR, "Virtq statistics is not supported for device %s.",
 416                         vdev->device->name);
 417                 return -ENOTSUP;
 418         }
 419         return mlx5_vdpa_virtq_stats_reset(priv, qid);
 420 }
 421
 422 static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {
 423         .get_queue_num = mlx5_vdpa_get_queue_num,
 424         .get_features = mlx5_vdpa_get_vdpa_features,
 425         .get_protocol_features = mlx5_vdpa_get_protocol_features,
 426         .dev_conf = mlx5_vdpa_dev_config,
 427         .dev_close = mlx5_vdpa_dev_close,
 428         .set_vring_state = mlx5_vdpa_set_vring_state,
 429         .set_features = mlx5_vdpa_features_set,
 430         .migration_done = NULL,
 431         .get_vfio_group_fd = NULL,
 432         .get_vfio_device_fd = mlx5_vdpa_get_device_fd,
 433         .get_notify_area = mlx5_vdpa_get_notify_area,
 434         .get_stats_names = mlx5_vdpa_get_stats_names,
 435         .get_stats = mlx5_vdpa_get_stats,
 436         .reset_stats = mlx5_vdpa_reset_stats,
 437 };
 438
 439 static int
 440 mlx5_vdpa_args_check_handler(const char *key, const char *val, void *opaque)
 441 {
 442         struct mlx5_vdpa_priv *priv = opaque;
 443         unsigned long tmp;
 444         int n_cores = sysconf(_SC_NPROCESSORS_ONLN);
 445
 446         errno = 0;
 447         tmp = strtoul(val, NULL, 0);
 448         if (errno) {
 449                 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 450                 return -errno;
 451         }
 452         if (strcmp(key, "event_mode") == 0) {
 453                 if (tmp <= MLX5_VDPA_EVENT_MODE_ONLY_INTERRUPT)
 454                         priv->event_mode = (int)tmp;
 455                 else
 456                         DRV_LOG(WARNING, "Invalid event_mode %s.", val);
 457         } else if (strcmp(key, "event_us") == 0) {
 458                 priv->event_us = (uint32_t)tmp;
 459         } else if (strcmp(key, "no_traffic_time") == 0) {
 460                 priv->no_traffic_max = (uint32_t)tmp;
 461         } else if (strcmp(key, "event_core") == 0) {
 462                 if (tmp >= (unsigned long)n_cores)
 463                         DRV_LOG(WARNING, "Invalid event_core %s.", val);
 464                 else
 465                         priv->event_core = tmp;
 466         } else if (strcmp(key, "hw_latency_mode") == 0) {
 467                 priv->hw_latency_mode = (uint32_t)tmp;
 468         } else if (strcmp(key, "hw_max_latency_us") == 0) {
 469                 priv->hw_max_latency_us = (uint32_t)tmp;
 470         } else if (strcmp(key, "hw_max_pending_comp") == 0) {
 471                 priv->hw_max_pending_comp = (uint32_t)tmp;
 472         }
 473         return 0;
 474 }
 475
 476 static void
 477 mlx5_vdpa_config_get(struct mlx5_kvargs_ctrl *mkvlist,
 478                      struct mlx5_vdpa_priv *priv)
 479 {
 480         const char **params = (const char *[]){
 481                 "event_core",
 482                 "event_mode",
 483                 "event_us",
 484                 "hw_latency_mode",
 485                 "hw_max_latency_us",
 486                 "hw_max_pending_comp",
 487                 "no_traffic_time",
 488                 NULL,
 489         };
 490
 491         priv->event_mode = MLX5_VDPA_EVENT_MODE_FIXED_TIMER;
 492         priv->event_us = 0;
 493         priv->event_core = -1;
 494         priv->no_traffic_max = MLX5_VDPA_DEFAULT_NO_TRAFFIC_MAX;
 495         if (mkvlist == NULL)
 496                 return;
 497         mlx5_kvargs_process(mkvlist, params, mlx5_vdpa_args_check_handler,
 498                             priv);
 499         if (!priv->event_us &&
 500             priv->event_mode == MLX5_VDPA_EVENT_MODE_DYNAMIC_TIMER)
 501                 priv->event_us = MLX5_VDPA_DEFAULT_TIMER_STEP_US;
 502         DRV_LOG(DEBUG, "event mode is %d.", priv->event_mode);
 503         DRV_LOG(DEBUG, "event_us is %u us.", priv->event_us);
 504         DRV_LOG(DEBUG, "no traffic max is %u.", priv->no_traffic_max);
 505 }
 506
 507 static int
 508 mlx5_vdpa_dev_probe(struct mlx5_common_device *cdev,
 509                     struct mlx5_kvargs_ctrl *mkvlist)
 510 {
 511         struct mlx5_vdpa_priv *priv = NULL;
 512         struct mlx5_hca_attr *attr = &cdev->config.hca_attr;
 513         int retry;
 514
 515         if (!attr->vdpa.valid || !attr->vdpa.max_num_virtio_queues) {
 516                 DRV_LOG(ERR, "Not enough capabilities to support vdpa, maybe "
 517                         "old FW/OFED version?");
 518                 rte_errno = ENOTSUP;
 519                 return -rte_errno;
 520         }
 521         if (!attr->vdpa.queue_counters_valid)
 522                 DRV_LOG(DEBUG, "No capability to support virtq statistics.");
 523         priv = rte_zmalloc("mlx5 vDPA device private", sizeof(*priv) +
 524                            sizeof(struct mlx5_vdpa_virtq) *
 525                            attr->vdpa.max_num_virtio_queues * 2,
 526                            RTE_CACHE_LINE_SIZE);
 527         if (!priv) {
 528                 DRV_LOG(ERR, "Failed to allocate private memory.");
 529                 rte_errno = ENOMEM;
 530                 return -rte_errno;
 531         }
 532         priv->caps = attr->vdpa;
 533         priv->log_max_rqt_size = attr->log_max_rqt_size;
 534         priv->num_lag_ports = attr->num_lag_ports;
 535         if (attr->num_lag_ports == 0)
 536                 priv->num_lag_ports = 1;
 537         priv->cdev = cdev;
 538         for (retry = 0; retry < 7; retry++) {
 539                 priv->var = mlx5_glue->dv_alloc_var(priv->cdev->ctx, 0);
 540                 if (priv->var != NULL)
 541                         break;
 542                 DRV_LOG(WARNING, "Failed to allocate VAR, retry %d.\n", retry);
 543                 /* Wait Qemu release VAR during vdpa restart, 0.1 sec based. */
 544                 usleep(100000U << retry);
 545         }
 546         if (!priv->var) {
 547                 DRV_LOG(ERR, "Failed to allocate VAR %u.", errno);
 548                 goto error;
 549         }
 550         priv->err_intr_handle =
 551                 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
 552         if (priv->err_intr_handle == NULL) {
 553                 DRV_LOG(ERR, "Fail to allocate intr_handle");
 554                 goto error;
 555         }
 556         priv->vdev = rte_vdpa_register_device(cdev->dev, &mlx5_vdpa_ops);
 557         if (priv->vdev == NULL) {
 558                 DRV_LOG(ERR, "Failed to register vDPA device.");
 559                 rte_errno = rte_errno ? rte_errno : EINVAL;
 560                 goto error;
 561         }
 562         mlx5_vdpa_config_get(mkvlist, priv);
 563         SLIST_INIT(&priv->mr_list);
 564         pthread_mutex_init(&priv->vq_config_lock, NULL);
 565         pthread_mutex_lock(&priv_list_lock);
 566         TAILQ_INSERT_TAIL(&priv_list, priv, next);
 567         pthread_mutex_unlock(&priv_list_lock);
 568         return 0;
 569
 570 error:
 571         if (priv) {
 572                 if (priv->var)
 573                         mlx5_glue->dv_free_var(priv->var);
 574                 rte_intr_instance_free(priv->err_intr_handle);
 575                 rte_free(priv);
 576         }
 577         return -rte_errno;
 578 }
 579
 580 static int
 581 mlx5_vdpa_dev_remove(struct mlx5_common_device *cdev)
 582 {
 583         struct mlx5_vdpa_priv *priv = NULL;
 584         int found = 0;
 585
 586         pthread_mutex_lock(&priv_list_lock);
 587         TAILQ_FOREACH(priv, &priv_list, next) {
 588                 if (priv->vdev->device == cdev->dev) {
 589                         found = 1;
 590                         break;
 591                 }
 592         }
 593         if (found)
 594                 TAILQ_REMOVE(&priv_list, priv, next);
 595         pthread_mutex_unlock(&priv_list_lock);
 596         if (found) {
 597                 if (priv->configured)
 598                         mlx5_vdpa_dev_close(priv->vid);
 599                 if (priv->var) {
 600                         mlx5_glue->dv_free_var(priv->var);
 601                         priv->var = NULL;
 602                 }
 603                 if (priv->vdev)
 604                         rte_vdpa_unregister_device(priv->vdev);
 605                 pthread_mutex_destroy(&priv->vq_config_lock);
 606                 rte_intr_instance_free(priv->err_intr_handle);
 607                 rte_free(priv);
 608         }
 609         return 0;
 610 }
 611
 612 static const struct rte_pci_id mlx5_vdpa_pci_id_map[] = {
 613         {
 614                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 615                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 616         },
 617         {
 618                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 619                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF)
 620         },
 621         {
 622                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 623                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
 624         },
 625         {
 626                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 627                                 PCI_DEVICE_ID_MELLANOX_CONNECTXVF)
 628         },
 629         {
 630                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 631                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
 632         },
 633         {
 634                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 635                                 PCI_DEVICE_ID_MELLANOX_CONNECTX7)
 636         },
 637         {
 638                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 639                                 PCI_DEVICE_ID_MELLANOX_CONNECTX7BF)
 640         },
 641         {
 642                 .vendor_id = 0
 643         }
 644 };
 645
 646 static struct mlx5_class_driver mlx5_vdpa_driver = {
 647         .drv_class = MLX5_CLASS_VDPA,
 648         .name = RTE_STR(MLX5_VDPA_DRIVER_NAME),
 649         .id_table = mlx5_vdpa_pci_id_map,
 650         .probe = mlx5_vdpa_dev_probe,
 651         .remove = mlx5_vdpa_dev_remove,
 652 };
 653
 654 RTE_LOG_REGISTER_DEFAULT(mlx5_vdpa_logtype, NOTICE)
 655
 656 /**
 657  * Driver initialization routine.
 658  */
 659 RTE_INIT(rte_mlx5_vdpa_init)
 660 {
 661         mlx5_common_init();
 662         if (mlx5_glue)
 663                 mlx5_class_driver_register(&mlx5_vdpa_driver);
 664 }
 665
 666 RTE_PMD_EXPORT_NAME(MLX5_VDPA_DRIVER_NAME, __COUNTER__);
 667 RTE_PMD_REGISTER_PCI_TABLE(MLX5_VDPA_DRIVER_NAME, mlx5_vdpa_pci_id_map);
 668 RTE_PMD_REGISTER_KMOD_DEP(MLX5_VDPA_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");