drivers/vdpa/mlx5/mlx5_vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <unistd.h>
   5
   6 #include <rte_malloc.h>
   7 #include <rte_log.h>
   8 #include <rte_errno.h>
   9 #include <rte_bus_pci.h>
  10 #include <rte_pci.h>
  11 #include <rte_string_fns.h>
  12
  13 #include <mlx5_glue.h>
  14 #include <mlx5_common.h>
  15 #include <mlx5_devx_cmds.h>
  16 #include <mlx5_prm.h>
  17 #include <mlx5_nl.h>
  18
  19 #include "mlx5_vdpa_utils.h"
  20 #include "mlx5_vdpa.h"
  21
  22
  23 #define MLX5_VDPA_DEFAULT_FEATURES ((1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
  24                             (1ULL << VIRTIO_F_ANY_LAYOUT) | \
  25                             (1ULL << VIRTIO_NET_F_MQ) | \
  26                             (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \
  27                             (1ULL << VIRTIO_F_ORDER_PLATFORM) | \
  28                             (1ULL << VHOST_F_LOG_ALL))
  29
  30 #define MLX5_VDPA_PROTOCOL_FEATURES \
  31                             ((1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
  32                              (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
  33                              (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
  34                              (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) | \
  35                              (1ULL << VHOST_USER_PROTOCOL_F_MQ))
  36
  37 TAILQ_HEAD(mlx5_vdpa_privs, mlx5_vdpa_priv) priv_list =
  38                                               TAILQ_HEAD_INITIALIZER(priv_list);
  39 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
  40 int mlx5_vdpa_logtype;
  41
  42 static struct mlx5_vdpa_priv *
  43 mlx5_vdpa_find_priv_resource_by_did(int did)
  44 {
  45         struct mlx5_vdpa_priv *priv;
  46         int found = 0;
  47
  48         pthread_mutex_lock(&priv_list_lock);
  49         TAILQ_FOREACH(priv, &priv_list, next) {
  50                 if (did == priv->id) {
  51                         found = 1;
  52                         break;
  53                 }
  54         }
  55         pthread_mutex_unlock(&priv_list_lock);
  56         if (!found) {
  57                 DRV_LOG(ERR, "Invalid device id: %d.", did);
  58                 rte_errno = EINVAL;
  59                 return NULL;
  60         }
  61         return priv;
  62 }
  63
  64 static int
  65 mlx5_vdpa_get_queue_num(int did, uint32_t *queue_num)
  66 {
  67         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
  68
  69         if (priv == NULL) {
  70                 DRV_LOG(ERR, "Invalid device id: %d.", did);
  71                 return -1;
  72         }
  73         *queue_num = priv->caps.max_num_virtio_queues;
  74         return 0;
  75 }
  76
  77 static int
  78 mlx5_vdpa_get_vdpa_features(int did, uint64_t *features)
  79 {
  80         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
  81
  82         if (priv == NULL) {
  83                 DRV_LOG(ERR, "Invalid device id: %d.", did);
  84                 return -1;
  85         }
  86         *features = MLX5_VDPA_DEFAULT_FEATURES;
  87         if (priv->caps.virtio_queue_type & (1 << MLX5_VIRTQ_TYPE_PACKED))
  88                 *features |= (1ULL << VIRTIO_F_RING_PACKED);
  89         if (priv->caps.tso_ipv4)
  90                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO4);
  91         if (priv->caps.tso_ipv6)
  92                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO6);
  93         if (priv->caps.tx_csum)
  94                 *features |= (1ULL << VIRTIO_NET_F_CSUM);
  95         if (priv->caps.rx_csum)
  96                 *features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
  97         if (priv->caps.virtio_version_1_0)
  98                 *features |= (1ULL << VIRTIO_F_VERSION_1);
  99         return 0;
 100 }
 101
 102 static int
 103 mlx5_vdpa_get_protocol_features(int did, uint64_t *features)
 104 {
 105         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 106
 107         if (priv == NULL) {
 108                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 109                 return -1;
 110         }
 111         *features = MLX5_VDPA_PROTOCOL_FEATURES;
 112         return 0;
 113 }
 114
 115 static int
 116 mlx5_vdpa_set_vring_state(int vid, int vring, int state)
 117 {
 118         int did = rte_vhost_get_vdpa_device_id(vid);
 119         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 120
 121         if (priv == NULL) {
 122                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 123                 return -EINVAL;
 124         }
 125         if (vring >= (int)priv->caps.max_num_virtio_queues * 2) {
 126                 DRV_LOG(ERR, "Too big vring id: %d.", vring);
 127                 return -E2BIG;
 128         }
 129         return mlx5_vdpa_virtq_enable(priv, vring, state);
 130 }
 131
 132 static int
 133 mlx5_vdpa_direct_db_prepare(struct mlx5_vdpa_priv *priv)
 134 {
 135         int ret;
 136
 137         if (priv->direct_notifier) {
 138                 ret = rte_vhost_host_notifier_ctrl(priv->vid, false);
 139                 if (ret != 0) {
 140                         DRV_LOG(INFO, "Direct HW notifier FD cannot be "
 141                                 "destroyed for device %d: %d.", priv->vid, ret);
 142                         return -1;
 143                 }
 144                 priv->direct_notifier = 0;
 145         }
 146         ret = rte_vhost_host_notifier_ctrl(priv->vid, true);
 147         if (ret != 0)
 148                 DRV_LOG(INFO, "Direct HW notifier FD cannot be configured for"
 149                         " device %d: %d.", priv->vid, ret);
 150         else
 151                 priv->direct_notifier = 1;
 152         return 0;
 153 }
 154
 155 static int
 156 mlx5_vdpa_features_set(int vid)
 157 {
 158         int did = rte_vhost_get_vdpa_device_id(vid);
 159         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 160         uint64_t log_base, log_size;
 161         uint64_t features;
 162         int ret;
 163
 164         if (priv == NULL) {
 165                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 166                 return -EINVAL;
 167         }
 168         ret = rte_vhost_get_negotiated_features(vid, &features);
 169         if (ret) {
 170                 DRV_LOG(ERR, "Failed to get negotiated features.");
 171                 return ret;
 172         }
 173         if (RTE_VHOST_NEED_LOG(features)) {
 174                 ret = rte_vhost_get_log_base(vid, &log_base, &log_size);
 175                 if (ret) {
 176                         DRV_LOG(ERR, "Failed to get log base.");
 177                         return ret;
 178                 }
 179                 ret = mlx5_vdpa_dirty_bitmap_set(priv, log_base, log_size);
 180                 if (ret) {
 181                         DRV_LOG(ERR, "Failed to set dirty bitmap.");
 182                         return ret;
 183                 }
 184                 DRV_LOG(INFO, "mlx5 vdpa: enabling dirty logging...");
 185                 ret = mlx5_vdpa_logging_enable(priv, 1);
 186                 if (ret) {
 187                         DRV_LOG(ERR, "Failed t enable dirty logging.");
 188                         return ret;
 189                 }
 190         }
 191         return 0;
 192 }
 193
 194 static int
 195 mlx5_vdpa_dev_close(int vid)
 196 {
 197         int did = rte_vhost_get_vdpa_device_id(vid);
 198         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 199         int ret = 0;
 200
 201         if (priv == NULL) {
 202                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 203                 return -1;
 204         }
 205         if (priv->configured)
 206                 ret |= mlx5_vdpa_lm_log(priv);
 207         mlx5_vdpa_cqe_event_unset(priv);
 208         mlx5_vdpa_steer_unset(priv);
 209         mlx5_vdpa_virtqs_release(priv);
 210         mlx5_vdpa_event_qp_global_release(priv);
 211         mlx5_vdpa_mem_dereg(priv);
 212         priv->configured = 0;
 213         priv->vid = 0;
 214         DRV_LOG(INFO, "vDPA device %d was closed.", vid);
 215         return ret;
 216 }
 217
 218 static int
 219 mlx5_vdpa_dev_config(int vid)
 220 {
 221         int did = rte_vhost_get_vdpa_device_id(vid);
 222         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 223
 224         if (priv == NULL) {
 225                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 226                 return -EINVAL;
 227         }
 228         if (priv->configured && mlx5_vdpa_dev_close(vid)) {
 229                 DRV_LOG(ERR, "Failed to reconfigure vid %d.", vid);
 230                 return -1;
 231         }
 232         priv->vid = vid;
 233         if (mlx5_vdpa_mem_register(priv) || mlx5_vdpa_direct_db_prepare(priv) ||
 234             mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) ||
 235             mlx5_vdpa_cqe_event_setup(priv)) {
 236                 mlx5_vdpa_dev_close(vid);
 237                 return -1;
 238         }
 239         priv->configured = 1;
 240         DRV_LOG(INFO, "vDPA device %d was configured.", vid);
 241         return 0;
 242 }
 243
 244 static int
 245 mlx5_vdpa_get_device_fd(int vid)
 246 {
 247         int did = rte_vhost_get_vdpa_device_id(vid);
 248         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 249
 250         if (priv == NULL) {
 251                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 252                 return -EINVAL;
 253         }
 254         return priv->ctx->cmd_fd;
 255 }
 256
 257 static int
 258 mlx5_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
 259 {
 260         int did = rte_vhost_get_vdpa_device_id(vid);
 261         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 262
 263         RTE_SET_USED(qid);
 264         if (priv == NULL) {
 265                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 266                 return -EINVAL;
 267         }
 268         if (!priv->var) {
 269                 DRV_LOG(ERR, "VAR was not created for device %d, is the device"
 270                         " configured?.", did);
 271                 return -EINVAL;
 272         }
 273         *offset = priv->var->mmap_off;
 274         *size = priv->var->length;
 275         return 0;
 276 }
 277
 278 static int
 279 mlx5_vdpa_get_stats_names(int did, struct rte_vdpa_stat_name *stats_names,
 280                           unsigned int size)
 281 {
 282         static const char *mlx5_vdpa_stats_names[MLX5_VDPA_STATS_MAX] = {
 283                 "received_descriptors",
 284                 "completed_descriptors",
 285                 "bad descriptor errors",
 286                 "exceed max chain",
 287                 "invalid buffer",
 288                 "completion errors",
 289         };
 290         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 291         unsigned int i;
 292
 293         if (priv == NULL) {
 294                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 295                 return -ENODEV;
 296         }
 297         if (!stats_names)
 298                 return MLX5_VDPA_STATS_MAX;
 299         size = RTE_MIN(size, (unsigned int)MLX5_VDPA_STATS_MAX);
 300         for (i = 0; i < size; ++i)
 301                 strlcpy(stats_names[i].name, mlx5_vdpa_stats_names[i],
 302                         RTE_VDPA_STATS_NAME_SIZE);
 303         return size;
 304 }
 305
 306 static int
 307 mlx5_vdpa_get_stats(int did, int qid, struct rte_vdpa_stat *stats,
 308                     unsigned int n)
 309 {
 310         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 311
 312         if (priv == NULL) {
 313                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 314                 return -ENODEV;
 315         }
 316         if (!priv->configured) {
 317                 DRV_LOG(ERR, "Device %d was not configured.", did);
 318                 return -ENODATA;
 319         }
 320         if (qid >= (int)priv->nr_virtqs) {
 321                 DRV_LOG(ERR, "Too big vring id: %d.", qid);
 322                 return -E2BIG;
 323         }
 324         if (!priv->caps.queue_counters_valid) {
 325                 DRV_LOG(ERR, "Virtq statistics is not supported for device %d.",
 326                         did);
 327                 return -ENOTSUP;
 328         }
 329         return mlx5_vdpa_virtq_stats_get(priv, qid, stats, n);
 330 }
 331
 332 static int
 333 mlx5_vdpa_reset_stats(int did, int qid)
 334 {
 335         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 336
 337         if (priv == NULL) {
 338                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 339                 return -ENODEV;
 340         }
 341         if (!priv->configured) {
 342                 DRV_LOG(ERR, "Device %d was not configured.", did);
 343                 return -ENODATA;
 344         }
 345         if (qid >= (int)priv->nr_virtqs) {
 346                 DRV_LOG(ERR, "Too big vring id: %d.", qid);
 347                 return -E2BIG;
 348         }
 349         if (!priv->caps.queue_counters_valid) {
 350                 DRV_LOG(ERR, "Virtq statistics is not supported for device %d.",
 351                         did);
 352                 return -ENOTSUP;
 353         }
 354         return mlx5_vdpa_virtq_stats_reset(priv, qid);
 355 }
 356
 357 static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {
 358         .get_queue_num = mlx5_vdpa_get_queue_num,
 359         .get_features = mlx5_vdpa_get_vdpa_features,
 360         .get_protocol_features = mlx5_vdpa_get_protocol_features,
 361         .dev_conf = mlx5_vdpa_dev_config,
 362         .dev_close = mlx5_vdpa_dev_close,
 363         .set_vring_state = mlx5_vdpa_set_vring_state,
 364         .set_features = mlx5_vdpa_features_set,
 365         .migration_done = NULL,
 366         .get_vfio_group_fd = NULL,
 367         .get_vfio_device_fd = mlx5_vdpa_get_device_fd,
 368         .get_notify_area = mlx5_vdpa_get_notify_area,
 369         .get_stats_names = mlx5_vdpa_get_stats_names,
 370         .get_stats = mlx5_vdpa_get_stats,
 371         .reset_stats = mlx5_vdpa_reset_stats,
 372 };
 373
 374 static struct ibv_device *
 375 mlx5_vdpa_get_ib_device_match(struct rte_pci_addr *addr)
 376 {
 377         int n;
 378         struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
 379         struct ibv_device *ibv_match = NULL;
 380
 381         if (!ibv_list) {
 382                 rte_errno = ENOSYS;
 383                 return NULL;
 384         }
 385         while (n-- > 0) {
 386                 struct rte_pci_addr pci_addr;
 387
 388                 DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
 389                 if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &pci_addr))
 390                         continue;
 391                 if (rte_pci_addr_cmp(addr, &pci_addr))
 392                         continue;
 393                 ibv_match = ibv_list[n];
 394                 break;
 395         }
 396         if (!ibv_match)
 397                 rte_errno = ENOENT;
 398         mlx5_glue->free_device_list(ibv_list);
 399         return ibv_match;
 400 }
 401
 402 /* Try to disable ROCE by Netlink\Devlink. */
 403 static int
 404 mlx5_vdpa_nl_roce_disable(const char *addr)
 405 {
 406         int nlsk_fd = mlx5_nl_init(NETLINK_GENERIC);
 407         int devlink_id;
 408         int enable;
 409         int ret;
 410
 411         if (nlsk_fd < 0)
 412                 return nlsk_fd;
 413         devlink_id = mlx5_nl_devlink_family_id_get(nlsk_fd);
 414         if (devlink_id < 0) {
 415                 ret = devlink_id;
 416                 DRV_LOG(DEBUG, "Failed to get devlink id for ROCE operations by"
 417                         " Netlink.");
 418                 goto close;
 419         }
 420         ret = mlx5_nl_enable_roce_get(nlsk_fd, devlink_id, addr, &enable);
 421         if (ret) {
 422                 DRV_LOG(DEBUG, "Failed to get ROCE enable by Netlink: %d.",
 423                         ret);
 424                 goto close;
 425         } else if (!enable) {
 426                 DRV_LOG(INFO, "ROCE has already disabled(Netlink).");
 427                 goto close;
 428         }
 429         ret = mlx5_nl_enable_roce_set(nlsk_fd, devlink_id, addr, 0);
 430         if (ret)
 431                 DRV_LOG(DEBUG, "Failed to disable ROCE by Netlink: %d.", ret);
 432         else
 433                 DRV_LOG(INFO, "ROCE is disabled by Netlink successfully.");
 434 close:
 435         close(nlsk_fd);
 436         return ret;
 437 }
 438
 439 /* Try to disable ROCE by sysfs. */
 440 static int
 441 mlx5_vdpa_sys_roce_disable(const char *addr)
 442 {
 443         FILE *file_o;
 444         int enable;
 445         int ret;
 446
 447         MKSTR(file_p, "/sys/bus/pci/devices/%s/roce_enable", addr);
 448         file_o = fopen(file_p, "rb");
 449         if (!file_o) {
 450                 rte_errno = ENOTSUP;
 451                 return -ENOTSUP;
 452         }
 453         ret = fscanf(file_o, "%d", &enable);
 454         if (ret != 1) {
 455                 rte_errno = EINVAL;
 456                 ret = EINVAL;
 457                 goto close;
 458         } else if (!enable) {
 459                 ret = 0;
 460                 DRV_LOG(INFO, "ROCE has already disabled(sysfs).");
 461                 goto close;
 462         }
 463         fclose(file_o);
 464         file_o = fopen(file_p, "wb");
 465         if (!file_o) {
 466                 rte_errno = ENOTSUP;
 467                 return -ENOTSUP;
 468         }
 469         fprintf(file_o, "0\n");
 470         ret = 0;
 471 close:
 472         if (ret)
 473                 DRV_LOG(DEBUG, "Failed to disable ROCE by sysfs: %d.", ret);
 474         else
 475                 DRV_LOG(INFO, "ROCE is disabled by sysfs successfully.");
 476         fclose(file_o);
 477         return ret;
 478 }
 479
 480 #define MLX5_VDPA_MAX_RETRIES 20
 481 #define MLX5_VDPA_USEC 1000
 482 static int
 483 mlx5_vdpa_roce_disable(struct rte_pci_addr *addr, struct ibv_device **ibv)
 484 {
 485         char addr_name[64] = {0};
 486
 487         rte_pci_device_name(addr, addr_name, sizeof(addr_name));
 488         /* Firstly try to disable ROCE by Netlink and fallback to sysfs. */
 489         if (mlx5_vdpa_nl_roce_disable(addr_name) == 0 ||
 490             mlx5_vdpa_sys_roce_disable(addr_name) == 0) {
 491                 /*
 492                  * Succeed to disable ROCE, wait for the IB device to appear
 493                  * again after reload.
 494                  */
 495                 int r;
 496                 struct ibv_device *ibv_new;
 497
 498                 for (r = MLX5_VDPA_MAX_RETRIES; r; r--) {
 499                         ibv_new = mlx5_vdpa_get_ib_device_match(addr);
 500                         if (ibv_new) {
 501                                 *ibv = ibv_new;
 502                                 return 0;
 503                         }
 504                         usleep(MLX5_VDPA_USEC);
 505                 }
 506                 DRV_LOG(ERR, "Cannot much device %s after ROCE disable, "
 507                         "retries exceed %d", addr_name, MLX5_VDPA_MAX_RETRIES);
 508                 rte_errno = EAGAIN;
 509         }
 510         return -rte_errno;
 511 }
 512
 513 /**
 514  * DPDK callback to register a PCI device.
 515  *
 516  * This function spawns vdpa device out of a given PCI device.
 517  *
 518  * @param[in] pci_drv
 519  *   PCI driver structure (mlx5_vpda_driver).
 520  * @param[in] pci_dev
 521  *   PCI device information.
 522  *
 523  * @return
 524  *   0 on success, 1 to skip this driver, a negative errno value otherwise
 525  *   and rte_errno is set.
 526  */
 527 static int
 528 mlx5_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 529                     struct rte_pci_device *pci_dev __rte_unused)
 530 {
 531         struct ibv_device *ibv;
 532         struct mlx5_vdpa_priv *priv = NULL;
 533         struct ibv_context *ctx = NULL;
 534         struct mlx5_hca_attr attr;
 535         int ret;
 536
 537         if (mlx5_class_get(pci_dev->device.devargs) != MLX5_CLASS_VDPA) {
 538                 DRV_LOG(DEBUG, "Skip probing - should be probed by other mlx5"
 539                         " driver.");
 540                 return 1;
 541         }
 542         ibv = mlx5_vdpa_get_ib_device_match(&pci_dev->addr);
 543         if (!ibv) {
 544                 DRV_LOG(ERR, "No matching IB device for PCI slot "
 545                         PCI_PRI_FMT ".", pci_dev->addr.domain,
 546                         pci_dev->addr.bus, pci_dev->addr.devid,
 547                         pci_dev->addr.function);
 548                 return -rte_errno;
 549         } else {
 550                 DRV_LOG(INFO, "PCI information matches for device \"%s\".",
 551                         ibv->name);
 552         }
 553         if (mlx5_vdpa_roce_disable(&pci_dev->addr, &ibv) != 0) {
 554                 DRV_LOG(WARNING, "Failed to disable ROCE for \"%s\".",
 555                         ibv->name);
 556                 return -rte_errno;
 557         }
 558         ctx = mlx5_glue->dv_open_device(ibv);
 559         if (!ctx) {
 560                 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
 561                 rte_errno = ENODEV;
 562                 return -rte_errno;
 563         }
 564         ret = mlx5_devx_cmd_query_hca_attr(ctx, &attr);
 565         if (ret) {
 566                 DRV_LOG(ERR, "Unable to read HCA capabilities.");
 567                 rte_errno = ENOTSUP;
 568                 goto error;
 569         } else if (!attr.vdpa.valid || !attr.vdpa.max_num_virtio_queues) {
 570                 DRV_LOG(ERR, "Not enough capabilities to support vdpa, maybe "
 571                         "old FW/OFED version?");
 572                 rte_errno = ENOTSUP;
 573                 goto error;
 574         }
 575         if (!attr.vdpa.queue_counters_valid)
 576                 DRV_LOG(DEBUG, "No capability to support virtq statistics.");
 577         priv = rte_zmalloc("mlx5 vDPA device private", sizeof(*priv) +
 578                            sizeof(struct mlx5_vdpa_virtq) *
 579                            attr.vdpa.max_num_virtio_queues * 2,
 580                            RTE_CACHE_LINE_SIZE);
 581         if (!priv) {
 582                 DRV_LOG(ERR, "Failed to allocate private memory.");
 583                 rte_errno = ENOMEM;
 584                 goto error;
 585         }
 586         priv->caps = attr.vdpa;
 587         priv->log_max_rqt_size = attr.log_max_rqt_size;
 588         priv->ctx = ctx;
 589         priv->dev_addr.pci_addr = pci_dev->addr;
 590         priv->dev_addr.type = VDPA_ADDR_PCI;
 591         priv->var = mlx5_glue->dv_alloc_var(ctx, 0);
 592         if (!priv->var) {
 593                 DRV_LOG(ERR, "Failed to allocate VAR %u.\n", errno);
 594                 goto error;
 595         }
 596         priv->id = rte_vdpa_register_device(&priv->dev_addr, &mlx5_vdpa_ops);
 597         if (priv->id < 0) {
 598                 DRV_LOG(ERR, "Failed to register vDPA device.");
 599                 rte_errno = rte_errno ? rte_errno : EINVAL;
 600                 goto error;
 601         }
 602         SLIST_INIT(&priv->mr_list);
 603         pthread_mutex_lock(&priv_list_lock);
 604         TAILQ_INSERT_TAIL(&priv_list, priv, next);
 605         pthread_mutex_unlock(&priv_list_lock);
 606         return 0;
 607
 608 error:
 609         if (priv) {
 610                 if (priv->var)
 611                         mlx5_glue->dv_free_var(priv->var);
 612                 rte_free(priv);
 613         }
 614         if (ctx)
 615                 mlx5_glue->close_device(ctx);
 616         return -rte_errno;
 617 }
 618
 619 /**
 620  * DPDK callback to remove a PCI device.
 621  *
 622  * This function removes all vDPA devices belong to a given PCI device.
 623  *
 624  * @param[in] pci_dev
 625  *   Pointer to the PCI device.
 626  *
 627  * @return
 628  *   0 on success, the function cannot fail.
 629  */
 630 static int
 631 mlx5_vdpa_pci_remove(struct rte_pci_device *pci_dev)
 632 {
 633         struct mlx5_vdpa_priv *priv = NULL;
 634         int found = 0;
 635
 636         pthread_mutex_lock(&priv_list_lock);
 637         TAILQ_FOREACH(priv, &priv_list, next) {
 638                 if (memcmp(&priv->dev_addr.pci_addr, &pci_dev->addr,
 639                            sizeof(pci_dev->addr)) == 0) {
 640                         found = 1;
 641                         break;
 642                 }
 643         }
 644         if (found)
 645                 TAILQ_REMOVE(&priv_list, priv, next);
 646         pthread_mutex_unlock(&priv_list_lock);
 647         if (found) {
 648                 if (priv->configured)
 649                         mlx5_vdpa_dev_close(priv->vid);
 650                 if (priv->var) {
 651                         mlx5_glue->dv_free_var(priv->var);
 652                         priv->var = NULL;
 653                 }
 654                 mlx5_glue->close_device(priv->ctx);
 655                 rte_free(priv);
 656         }
 657         return 0;
 658 }
 659
 660 static const struct rte_pci_id mlx5_vdpa_pci_id_map[] = {
 661         {
 662                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 663                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 664         },
 665         {
 666                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 667                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF)
 668         },
 669         {
 670                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 671                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
 672         },
 673         {
 674                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 675                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF)
 676         },
 677         {
 678                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 679                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
 680         },
 681         {
 682                 .vendor_id = 0
 683         }
 684 };
 685
 686 static struct rte_pci_driver mlx5_vdpa_driver = {
 687         .driver = {
 688                 .name = "mlx5_vdpa",
 689         },
 690         .id_table = mlx5_vdpa_pci_id_map,
 691         .probe = mlx5_vdpa_pci_probe,
 692         .remove = mlx5_vdpa_pci_remove,
 693         .drv_flags = 0,
 694 };
 695
 696 /**
 697  * Driver initialization routine.
 698  */
 699 RTE_INIT(rte_mlx5_vdpa_init)
 700 {
 701         /* Initialize common log type. */
 702         mlx5_vdpa_logtype = rte_log_register("pmd.vdpa.mlx5");
 703         if (mlx5_vdpa_logtype >= 0)
 704                 rte_log_set_level(mlx5_vdpa_logtype, RTE_LOG_NOTICE);
 705         if (mlx5_glue)
 706                 rte_pci_register(&mlx5_vdpa_driver);
 707 }
 708
 709 RTE_PMD_EXPORT_NAME(net_mlx5_vdpa, __COUNTER__);
 710 RTE_PMD_REGISTER_PCI_TABLE(net_mlx5_vdpa, mlx5_vdpa_pci_id_map);
 711 RTE_PMD_REGISTER_KMOD_DEP(net_mlx5_vdpa, "* ib_uverbs & mlx5_core & mlx5_ib");