drivers/vdpa/mlx5/mlx5_vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <unistd.h>
   5 #include <net/if.h>
   6 #include <sys/socket.h>
   7 #include <sys/ioctl.h>
   8 #include <fcntl.h>
   9 #include <netinet/in.h>
  10
  11 #include <rte_malloc.h>
  12 #include <rte_log.h>
  13 #include <rte_errno.h>
  14 #include <rte_string_fns.h>
  15 #include <rte_bus_pci.h>
  16
  17 #include <mlx5_glue.h>
  18 #include <mlx5_common.h>
  19 #include <mlx5_common_defs.h>
  20 #include <mlx5_devx_cmds.h>
  21 #include <mlx5_prm.h>
  22 #include <mlx5_nl.h>
  23
  24 #include "mlx5_vdpa_utils.h"
  25 #include "mlx5_vdpa.h"
  26
  27 #define MLX5_VDPA_DRIVER_NAME vdpa_mlx5
  28
  29 #define MLX5_VDPA_DEFAULT_FEATURES ((1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
  30                             (1ULL << VIRTIO_F_ANY_LAYOUT) | \
  31                             (1ULL << VIRTIO_NET_F_MQ) | \
  32                             (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \
  33                             (1ULL << VIRTIO_F_ORDER_PLATFORM) | \
  34                             (1ULL << VHOST_F_LOG_ALL) | \
  35                             (1ULL << VIRTIO_NET_F_MTU))
  36
  37 #define MLX5_VDPA_PROTOCOL_FEATURES \
  38                             ((1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
  39                              (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
  40                              (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
  41                              (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) | \
  42                              (1ULL << VHOST_USER_PROTOCOL_F_MQ) | \
  43                              (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU) | \
  44                              (1ULL << VHOST_USER_PROTOCOL_F_STATUS))
  45
  46 #define MLX5_VDPA_DEFAULT_NO_TRAFFIC_MAX 16LLU
  47
  48 TAILQ_HEAD(mlx5_vdpa_privs, mlx5_vdpa_priv) priv_list =
  49                                               TAILQ_HEAD_INITIALIZER(priv_list);
  50 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
  51
  52 static struct mlx5_vdpa_priv *
  53 mlx5_vdpa_find_priv_resource_by_vdev(struct rte_vdpa_device *vdev)
  54 {
  55         struct mlx5_vdpa_priv *priv;
  56         int found = 0;
  57
  58         pthread_mutex_lock(&priv_list_lock);
  59         TAILQ_FOREACH(priv, &priv_list, next) {
  60                 if (vdev == priv->vdev) {
  61                         found = 1;
  62                         break;
  63                 }
  64         }
  65         pthread_mutex_unlock(&priv_list_lock);
  66         if (!found) {
  67                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
  68                 rte_errno = EINVAL;
  69                 return NULL;
  70         }
  71         return priv;
  72 }
  73
  74 static int
  75 mlx5_vdpa_get_queue_num(struct rte_vdpa_device *vdev, uint32_t *queue_num)
  76 {
  77         struct mlx5_vdpa_priv *priv =
  78                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
  79
  80         if (priv == NULL) {
  81                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
  82                 return -1;
  83         }
  84         *queue_num = priv->caps.max_num_virtio_queues;
  85         return 0;
  86 }
  87
  88 static int
  89 mlx5_vdpa_get_vdpa_features(struct rte_vdpa_device *vdev, uint64_t *features)
  90 {
  91         struct mlx5_vdpa_priv *priv =
  92                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
  93
  94         if (priv == NULL) {
  95                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
  96                 return -1;
  97         }
  98         *features = MLX5_VDPA_DEFAULT_FEATURES;
  99         if (priv->caps.virtio_queue_type & (1 << MLX5_VIRTQ_TYPE_PACKED))
 100                 *features |= (1ULL << VIRTIO_F_RING_PACKED);
 101         if (priv->caps.tso_ipv4)
 102                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO4);
 103         if (priv->caps.tso_ipv6)
 104                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO6);
 105         if (priv->caps.tx_csum)
 106                 *features |= (1ULL << VIRTIO_NET_F_CSUM);
 107         if (priv->caps.rx_csum)
 108                 *features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
 109         if (priv->caps.virtio_version_1_0)
 110                 *features |= (1ULL << VIRTIO_F_VERSION_1);
 111         return 0;
 112 }
 113
 114 static int
 115 mlx5_vdpa_get_protocol_features(struct rte_vdpa_device *vdev,
 116                 uint64_t *features)
 117 {
 118         struct mlx5_vdpa_priv *priv =
 119                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 120
 121         if (priv == NULL) {
 122                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 123                 return -1;
 124         }
 125         *features = MLX5_VDPA_PROTOCOL_FEATURES;
 126         return 0;
 127 }
 128
 129 static int
 130 mlx5_vdpa_set_vring_state(int vid, int vring, int state)
 131 {
 132         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 133         struct mlx5_vdpa_priv *priv =
 134                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 135         int ret;
 136
 137         if (priv == NULL) {
 138                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 139                 return -EINVAL;
 140         }
 141         if (vring >= (int)priv->caps.max_num_virtio_queues * 2) {
 142                 DRV_LOG(ERR, "Too big vring id: %d.", vring);
 143                 return -E2BIG;
 144         }
 145         pthread_mutex_lock(&priv->vq_config_lock);
 146         ret = mlx5_vdpa_virtq_enable(priv, vring, state);
 147         pthread_mutex_unlock(&priv->vq_config_lock);
 148         return ret;
 149 }
 150
 151 static int
 152 mlx5_vdpa_features_set(int vid)
 153 {
 154         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 155         struct mlx5_vdpa_priv *priv =
 156                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 157         uint64_t log_base, log_size;
 158         uint64_t features;
 159         int ret;
 160
 161         if (priv == NULL) {
 162                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 163                 return -EINVAL;
 164         }
 165         ret = rte_vhost_get_negotiated_features(vid, &features);
 166         if (ret) {
 167                 DRV_LOG(ERR, "Failed to get negotiated features.");
 168                 return ret;
 169         }
 170         if (RTE_VHOST_NEED_LOG(features)) {
 171                 ret = rte_vhost_get_log_base(vid, &log_base, &log_size);
 172                 if (ret) {
 173                         DRV_LOG(ERR, "Failed to get log base.");
 174                         return ret;
 175                 }
 176                 ret = mlx5_vdpa_dirty_bitmap_set(priv, log_base, log_size);
 177                 if (ret) {
 178                         DRV_LOG(ERR, "Failed to set dirty bitmap.");
 179                         return ret;
 180                 }
 181                 DRV_LOG(INFO, "mlx5 vdpa: enabling dirty logging...");
 182                 ret = mlx5_vdpa_logging_enable(priv, 1);
 183                 if (ret) {
 184                         DRV_LOG(ERR, "Failed t enable dirty logging.");
 185                         return ret;
 186                 }
 187         }
 188         return 0;
 189 }
 190
 191 static int
 192 mlx5_vdpa_mtu_set(struct mlx5_vdpa_priv *priv)
 193 {
 194         struct ifreq request;
 195         uint16_t vhost_mtu = 0;
 196         uint16_t kern_mtu = 0;
 197         int ret = rte_vhost_get_mtu(priv->vid, &vhost_mtu);
 198         int sock;
 199         int retries = MLX5_VDPA_MAX_RETRIES;
 200
 201         if (ret) {
 202                 DRV_LOG(DEBUG, "Cannot get vhost MTU - %d.", ret);
 203                 return ret;
 204         }
 205         if (!vhost_mtu) {
 206                 DRV_LOG(DEBUG, "Vhost MTU is 0.");
 207                 return ret;
 208         }
 209         ret = mlx5_get_ifname_sysfs
 210                                 (mlx5_os_get_ctx_device_name(priv->cdev->ctx),
 211                                  request.ifr_name);
 212         if (ret) {
 213                 DRV_LOG(DEBUG, "Cannot get kernel IF name - %d.", ret);
 214                 return ret;
 215         }
 216         sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
 217         if (sock == -1) {
 218                 DRV_LOG(DEBUG, "Cannot open IF socket.");
 219                 return sock;
 220         }
 221         while (retries--) {
 222                 ret = ioctl(sock, SIOCGIFMTU, &request);
 223                 if (ret == -1)
 224                         break;
 225                 kern_mtu = request.ifr_mtu;
 226                 DRV_LOG(DEBUG, "MTU: current %d requested %d.", (int)kern_mtu,
 227                         (int)vhost_mtu);
 228                 if (kern_mtu == vhost_mtu)
 229                         break;
 230                 request.ifr_mtu = vhost_mtu;
 231                 ret = ioctl(sock, SIOCSIFMTU, &request);
 232                 if (ret == -1)
 233                         break;
 234                 request.ifr_mtu = 0;
 235                 usleep(MLX5_VDPA_USEC);
 236         }
 237         close(sock);
 238         return kern_mtu == vhost_mtu ? 0 : -1;
 239 }
 240
 241 static int
 242 mlx5_vdpa_dev_close(int vid)
 243 {
 244         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 245         struct mlx5_vdpa_priv *priv =
 246                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 247         int ret = 0;
 248
 249         if (priv == NULL) {
 250                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 251                 return -1;
 252         }
 253         mlx5_vdpa_err_event_unset(priv);
 254         mlx5_vdpa_cqe_event_unset(priv);
 255         if (priv->state == MLX5_VDPA_STATE_CONFIGURED) {
 256                 ret |= mlx5_vdpa_lm_log(priv);
 257                 priv->state = MLX5_VDPA_STATE_IN_PROGRESS;
 258         }
 259         mlx5_vdpa_steer_unset(priv);
 260         mlx5_vdpa_virtqs_release(priv);
 261         mlx5_vdpa_event_qp_global_release(priv);
 262         mlx5_vdpa_mem_dereg(priv);
 263         priv->state = MLX5_VDPA_STATE_PROBED;
 264         priv->vid = 0;
 265         /* The mutex may stay locked after event thread cancel - initiate it. */
 266         pthread_mutex_init(&priv->vq_config_lock, NULL);
 267         DRV_LOG(INFO, "vDPA device %d was closed.", vid);
 268         return ret;
 269 }
 270
 271 static int
 272 mlx5_vdpa_dev_config(int vid)
 273 {
 274         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 275         struct mlx5_vdpa_priv *priv =
 276                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 277
 278         if (priv == NULL) {
 279                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 280                 return -EINVAL;
 281         }
 282         if (priv->state == MLX5_VDPA_STATE_CONFIGURED &&
 283             mlx5_vdpa_dev_close(vid)) {
 284                 DRV_LOG(ERR, "Failed to reconfigure vid %d.", vid);
 285                 return -1;
 286         }
 287         priv->vid = vid;
 288         if (mlx5_vdpa_mtu_set(priv))
 289                 DRV_LOG(WARNING, "MTU cannot be set on device %s.",
 290                                 vdev->device->name);
 291         if (mlx5_vdpa_mem_register(priv) || mlx5_vdpa_err_event_setup(priv) ||
 292             mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) ||
 293             mlx5_vdpa_cqe_event_setup(priv)) {
 294                 mlx5_vdpa_dev_close(vid);
 295                 return -1;
 296         }
 297         priv->state = MLX5_VDPA_STATE_CONFIGURED;
 298         DRV_LOG(INFO, "vDPA device %d was configured.", vid);
 299         return 0;
 300 }
 301
 302 static int
 303 mlx5_vdpa_get_device_fd(int vid)
 304 {
 305         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 306         struct mlx5_vdpa_priv *priv =
 307                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 308
 309         if (priv == NULL) {
 310                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 311                 return -EINVAL;
 312         }
 313         return ((struct ibv_context *)priv->cdev->ctx)->cmd_fd;
 314 }
 315
 316 static int
 317 mlx5_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
 318 {
 319         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 320         struct mlx5_vdpa_priv *priv =
 321                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 322
 323         RTE_SET_USED(qid);
 324         if (priv == NULL) {
 325                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 326                 return -EINVAL;
 327         }
 328         if (!priv->var) {
 329                 DRV_LOG(ERR, "VAR was not created for device %s, is the device"
 330                         " configured?.", vdev->device->name);
 331                 return -EINVAL;
 332         }
 333         *offset = priv->var->mmap_off;
 334         *size = priv->var->length;
 335         return 0;
 336 }
 337
 338 static int
 339 mlx5_vdpa_get_stats_names(struct rte_vdpa_device *vdev,
 340                 struct rte_vdpa_stat_name *stats_names,
 341                 unsigned int size)
 342 {
 343         static const char *mlx5_vdpa_stats_names[MLX5_VDPA_STATS_MAX] = {
 344                 "received_descriptors",
 345                 "completed_descriptors",
 346                 "bad descriptor errors",
 347                 "exceed max chain",
 348                 "invalid buffer",
 349                 "completion errors",
 350         };
 351         struct mlx5_vdpa_priv *priv =
 352                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 353         unsigned int i;
 354
 355         if (priv == NULL) {
 356                 DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
 357                 return -ENODEV;
 358         }
 359         if (!stats_names)
 360                 return MLX5_VDPA_STATS_MAX;
 361         size = RTE_MIN(size, (unsigned int)MLX5_VDPA_STATS_MAX);
 362         for (i = 0; i < size; ++i)
 363                 strlcpy(stats_names[i].name, mlx5_vdpa_stats_names[i],
 364                         RTE_VDPA_STATS_NAME_SIZE);
 365         return size;
 366 }
 367
 368 static int
 369 mlx5_vdpa_get_stats(struct rte_vdpa_device *vdev, int qid,
 370                 struct rte_vdpa_stat *stats, unsigned int n)
 371 {
 372         struct mlx5_vdpa_priv *priv =
 373                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 374
 375         if (priv == NULL) {
 376                 DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
 377                 return -ENODEV;
 378         }
 379         if (priv->state == MLX5_VDPA_STATE_PROBED) {
 380                 DRV_LOG(ERR, "Device %s was not configured.",
 381                                 vdev->device->name);
 382                 return -ENODATA;
 383         }
 384         if (qid >= (int)priv->nr_virtqs) {
 385                 DRV_LOG(ERR, "Too big vring id: %d for device %s.", qid,
 386                                 vdev->device->name);
 387                 return -E2BIG;
 388         }
 389         if (!priv->caps.queue_counters_valid) {
 390                 DRV_LOG(ERR, "Virtq statistics is not supported for device %s.",
 391                         vdev->device->name);
 392                 return -ENOTSUP;
 393         }
 394         return mlx5_vdpa_virtq_stats_get(priv, qid, stats, n);
 395 }
 396
 397 static int
 398 mlx5_vdpa_reset_stats(struct rte_vdpa_device *vdev, int qid)
 399 {
 400         struct mlx5_vdpa_priv *priv =
 401                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 402
 403         if (priv == NULL) {
 404                 DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
 405                 return -ENODEV;
 406         }
 407         if (priv->state == MLX5_VDPA_STATE_PROBED) {
 408                 DRV_LOG(ERR, "Device %s was not configured.",
 409                                 vdev->device->name);
 410                 return -ENODATA;
 411         }
 412         if (qid >= (int)priv->nr_virtqs) {
 413                 DRV_LOG(ERR, "Too big vring id: %d for device %s.", qid,
 414                                 vdev->device->name);
 415                 return -E2BIG;
 416         }
 417         if (!priv->caps.queue_counters_valid) {
 418                 DRV_LOG(ERR, "Virtq statistics is not supported for device %s.",
 419                         vdev->device->name);
 420                 return -ENOTSUP;
 421         }
 422         return mlx5_vdpa_virtq_stats_reset(priv, qid);
 423 }
 424
 425 static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {
 426         .get_queue_num = mlx5_vdpa_get_queue_num,
 427         .get_features = mlx5_vdpa_get_vdpa_features,
 428         .get_protocol_features = mlx5_vdpa_get_protocol_features,
 429         .dev_conf = mlx5_vdpa_dev_config,
 430         .dev_close = mlx5_vdpa_dev_close,
 431         .set_vring_state = mlx5_vdpa_set_vring_state,
 432         .set_features = mlx5_vdpa_features_set,
 433         .migration_done = NULL,
 434         .get_vfio_group_fd = NULL,
 435         .get_vfio_device_fd = mlx5_vdpa_get_device_fd,
 436         .get_notify_area = mlx5_vdpa_get_notify_area,
 437         .get_stats_names = mlx5_vdpa_get_stats_names,
 438         .get_stats = mlx5_vdpa_get_stats,
 439         .reset_stats = mlx5_vdpa_reset_stats,
 440 };
 441
 442 static int
 443 mlx5_vdpa_args_check_handler(const char *key, const char *val, void *opaque)
 444 {
 445         struct mlx5_vdpa_priv *priv = opaque;
 446         unsigned long tmp;
 447         int n_cores = sysconf(_SC_NPROCESSORS_ONLN);
 448
 449         errno = 0;
 450         tmp = strtoul(val, NULL, 0);
 451         if (errno) {
 452                 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 453                 return -errno;
 454         }
 455         if (strcmp(key, "event_mode") == 0) {
 456                 if (tmp <= MLX5_VDPA_EVENT_MODE_ONLY_INTERRUPT)
 457                         priv->event_mode = (int)tmp;
 458                 else
 459                         DRV_LOG(WARNING, "Invalid event_mode %s.", val);
 460         } else if (strcmp(key, "event_us") == 0) {
 461                 priv->event_us = (uint32_t)tmp;
 462         } else if (strcmp(key, "no_traffic_time") == 0) {
 463                 priv->no_traffic_max = (uint32_t)tmp;
 464         } else if (strcmp(key, "event_core") == 0) {
 465                 if (tmp >= (unsigned long)n_cores)
 466                         DRV_LOG(WARNING, "Invalid event_core %s.", val);
 467                 else
 468                         priv->event_core = tmp;
 469         } else if (strcmp(key, "hw_latency_mode") == 0) {
 470                 priv->hw_latency_mode = (uint32_t)tmp;
 471         } else if (strcmp(key, "hw_max_latency_us") == 0) {
 472                 priv->hw_max_latency_us = (uint32_t)tmp;
 473         } else if (strcmp(key, "hw_max_pending_comp") == 0) {
 474                 priv->hw_max_pending_comp = (uint32_t)tmp;
 475         }
 476         return 0;
 477 }
 478
 479 static void
 480 mlx5_vdpa_config_get(struct mlx5_kvargs_ctrl *mkvlist,
 481                      struct mlx5_vdpa_priv *priv)
 482 {
 483         const char **params = (const char *[]){
 484                 "event_core",
 485                 "event_mode",
 486                 "event_us",
 487                 "hw_latency_mode",
 488                 "hw_max_latency_us",
 489                 "hw_max_pending_comp",
 490                 "no_traffic_time",
 491                 NULL,
 492         };
 493
 494         priv->event_mode = MLX5_VDPA_EVENT_MODE_FIXED_TIMER;
 495         priv->event_us = 0;
 496         priv->event_core = -1;
 497         priv->no_traffic_max = MLX5_VDPA_DEFAULT_NO_TRAFFIC_MAX;
 498         if (mkvlist == NULL)
 499                 return;
 500         mlx5_kvargs_process(mkvlist, params, mlx5_vdpa_args_check_handler,
 501                             priv);
 502         if (!priv->event_us &&
 503             priv->event_mode == MLX5_VDPA_EVENT_MODE_DYNAMIC_TIMER)
 504                 priv->event_us = MLX5_VDPA_DEFAULT_TIMER_STEP_US;
 505         DRV_LOG(DEBUG, "event mode is %d.", priv->event_mode);
 506         DRV_LOG(DEBUG, "event_us is %u us.", priv->event_us);
 507         DRV_LOG(DEBUG, "no traffic max is %u.", priv->no_traffic_max);
 508 }
 509
 510 static int
 511 mlx5_vdpa_dev_probe(struct mlx5_common_device *cdev,
 512                     struct mlx5_kvargs_ctrl *mkvlist)
 513 {
 514         struct mlx5_vdpa_priv *priv = NULL;
 515         struct mlx5_hca_attr *attr = &cdev->config.hca_attr;
 516         int retry;
 517
 518         if (!attr->vdpa.valid || !attr->vdpa.max_num_virtio_queues) {
 519                 DRV_LOG(ERR, "Not enough capabilities to support vdpa, maybe "
 520                         "old FW/OFED version?");
 521                 rte_errno = ENOTSUP;
 522                 return -rte_errno;
 523         }
 524         if (!attr->vdpa.queue_counters_valid)
 525                 DRV_LOG(DEBUG, "No capability to support virtq statistics.");
 526         priv = rte_zmalloc("mlx5 vDPA device private", sizeof(*priv) +
 527                            sizeof(struct mlx5_vdpa_virtq) *
 528                            attr->vdpa.max_num_virtio_queues * 2,
 529                            RTE_CACHE_LINE_SIZE);
 530         if (!priv) {
 531                 DRV_LOG(ERR, "Failed to allocate private memory.");
 532                 rte_errno = ENOMEM;
 533                 return -rte_errno;
 534         }
 535         priv->caps = attr->vdpa;
 536         priv->log_max_rqt_size = attr->log_max_rqt_size;
 537         priv->num_lag_ports = attr->num_lag_ports;
 538         if (attr->num_lag_ports == 0)
 539                 priv->num_lag_ports = 1;
 540         priv->cdev = cdev;
 541         for (retry = 0; retry < 7; retry++) {
 542                 priv->var = mlx5_glue->dv_alloc_var(priv->cdev->ctx, 0);
 543                 if (priv->var != NULL)
 544                         break;
 545                 DRV_LOG(WARNING, "Failed to allocate VAR, retry %d.\n", retry);
 546                 /* Wait Qemu release VAR during vdpa restart, 0.1 sec based. */
 547                 usleep(100000U << retry);
 548         }
 549         if (!priv->var) {
 550                 DRV_LOG(ERR, "Failed to allocate VAR %u.", errno);
 551                 goto error;
 552         }
 553         priv->err_intr_handle =
 554                 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
 555         if (priv->err_intr_handle == NULL) {
 556                 DRV_LOG(ERR, "Fail to allocate intr_handle");
 557                 goto error;
 558         }
 559         priv->vdev = rte_vdpa_register_device(cdev->dev, &mlx5_vdpa_ops);
 560         if (priv->vdev == NULL) {
 561                 DRV_LOG(ERR, "Failed to register vDPA device.");
 562                 rte_errno = rte_errno ? rte_errno : EINVAL;
 563                 goto error;
 564         }
 565         mlx5_vdpa_config_get(mkvlist, priv);
 566         SLIST_INIT(&priv->mr_list);
 567         pthread_mutex_init(&priv->vq_config_lock, NULL);
 568         pthread_mutex_lock(&priv_list_lock);
 569         TAILQ_INSERT_TAIL(&priv_list, priv, next);
 570         pthread_mutex_unlock(&priv_list_lock);
 571         return 0;
 572
 573 error:
 574         if (priv) {
 575                 if (priv->var)
 576                         mlx5_glue->dv_free_var(priv->var);
 577                 rte_intr_instance_free(priv->err_intr_handle);
 578                 rte_free(priv);
 579         }
 580         return -rte_errno;
 581 }
 582
 583 static int
 584 mlx5_vdpa_dev_remove(struct mlx5_common_device *cdev)
 585 {
 586         struct mlx5_vdpa_priv *priv = NULL;
 587         int found = 0;
 588
 589         pthread_mutex_lock(&priv_list_lock);
 590         TAILQ_FOREACH(priv, &priv_list, next) {
 591                 if (priv->vdev->device == cdev->dev) {
 592                         found = 1;
 593                         break;
 594                 }
 595         }
 596         if (found)
 597                 TAILQ_REMOVE(&priv_list, priv, next);
 598         pthread_mutex_unlock(&priv_list_lock);
 599         if (found) {
 600                 if (priv->state == MLX5_VDPA_STATE_CONFIGURED)
 601                         mlx5_vdpa_dev_close(priv->vid);
 602                 if (priv->var) {
 603                         mlx5_glue->dv_free_var(priv->var);
 604                         priv->var = NULL;
 605                 }
 606                 if (priv->vdev)
 607                         rte_vdpa_unregister_device(priv->vdev);
 608                 pthread_mutex_destroy(&priv->vq_config_lock);
 609                 rte_intr_instance_free(priv->err_intr_handle);
 610                 rte_free(priv);
 611         }
 612         return 0;
 613 }
 614
 615 static const struct rte_pci_id mlx5_vdpa_pci_id_map[] = {
 616         {
 617                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 618                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 619         },
 620         {
 621                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 622                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF)
 623         },
 624         {
 625                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 626                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
 627         },
 628         {
 629                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 630                                 PCI_DEVICE_ID_MELLANOX_CONNECTXVF)
 631         },
 632         {
 633                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 634                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
 635         },
 636         {
 637                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 638                                 PCI_DEVICE_ID_MELLANOX_CONNECTX7)
 639         },
 640         {
 641                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 642                                 PCI_DEVICE_ID_MELLANOX_CONNECTX7BF)
 643         },
 644         {
 645                 .vendor_id = 0
 646         }
 647 };
 648
 649 static struct mlx5_class_driver mlx5_vdpa_driver = {
 650         .drv_class = MLX5_CLASS_VDPA,
 651         .name = RTE_STR(MLX5_VDPA_DRIVER_NAME),
 652         .id_table = mlx5_vdpa_pci_id_map,
 653         .probe = mlx5_vdpa_dev_probe,
 654         .remove = mlx5_vdpa_dev_remove,
 655 };
 656
 657 RTE_LOG_REGISTER_DEFAULT(mlx5_vdpa_logtype, NOTICE)
 658
 659 /**
 660  * Driver initialization routine.
 661  */
 662 RTE_INIT(rte_mlx5_vdpa_init)
 663 {
 664         mlx5_common_init();
 665         if (mlx5_glue)
 666                 mlx5_class_driver_register(&mlx5_vdpa_driver);
 667 }
 668
 669 RTE_PMD_EXPORT_NAME(MLX5_VDPA_DRIVER_NAME, __COUNTER__);
 670 RTE_PMD_REGISTER_PCI_TABLE(MLX5_VDPA_DRIVER_NAME, mlx5_vdpa_pci_id_map);
 671 RTE_PMD_REGISTER_KMOD_DEP(MLX5_VDPA_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");