drivers/vdpa/mlx5/mlx5_vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <unistd.h>
   5
   6 #include <rte_malloc.h>
   7 #include <rte_log.h>
   8 #include <rte_errno.h>
   9 #include <rte_bus_pci.h>
  10 #include <rte_pci.h>
  11
  12 #include <mlx5_glue.h>
  13 #include <mlx5_common.h>
  14 #include <mlx5_devx_cmds.h>
  15 #include <mlx5_prm.h>
  16 #include <mlx5_nl.h>
  17
  18 #include "mlx5_vdpa_utils.h"
  19 #include "mlx5_vdpa.h"
  20
  21
  22 #define MLX5_VDPA_DEFAULT_FEATURES ((1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
  23                             (1ULL << VIRTIO_F_ANY_LAYOUT) | \
  24                             (1ULL << VIRTIO_NET_F_MQ) | \
  25                             (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \
  26                             (1ULL << VIRTIO_F_ORDER_PLATFORM) | \
  27                             (1ULL << VHOST_F_LOG_ALL))
  28
  29 #define MLX5_VDPA_PROTOCOL_FEATURES \
  30                             ((1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
  31                              (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
  32                              (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
  33                              (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) | \
  34                              (1ULL << VHOST_USER_PROTOCOL_F_MQ))
  35
  36 TAILQ_HEAD(mlx5_vdpa_privs, mlx5_vdpa_priv) priv_list =
  37                                               TAILQ_HEAD_INITIALIZER(priv_list);
  38 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
  39 int mlx5_vdpa_logtype;
  40
  41 static struct mlx5_vdpa_priv *
  42 mlx5_vdpa_find_priv_resource_by_did(int did)
  43 {
  44         struct mlx5_vdpa_priv *priv;
  45         int found = 0;
  46
  47         pthread_mutex_lock(&priv_list_lock);
  48         TAILQ_FOREACH(priv, &priv_list, next) {
  49                 if (did == priv->id) {
  50                         found = 1;
  51                         break;
  52                 }
  53         }
  54         pthread_mutex_unlock(&priv_list_lock);
  55         if (!found) {
  56                 DRV_LOG(ERR, "Invalid device id: %d.", did);
  57                 rte_errno = EINVAL;
  58                 return NULL;
  59         }
  60         return priv;
  61 }
  62
  63 static int
  64 mlx5_vdpa_get_queue_num(int did, uint32_t *queue_num)
  65 {
  66         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
  67
  68         if (priv == NULL) {
  69                 DRV_LOG(ERR, "Invalid device id: %d.", did);
  70                 return -1;
  71         }
  72         *queue_num = priv->caps.max_num_virtio_queues;
  73         return 0;
  74 }
  75
  76 static int
  77 mlx5_vdpa_get_vdpa_features(int did, uint64_t *features)
  78 {
  79         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
  80
  81         if (priv == NULL) {
  82                 DRV_LOG(ERR, "Invalid device id: %d.", did);
  83                 return -1;
  84         }
  85         *features = MLX5_VDPA_DEFAULT_FEATURES;
  86         if (priv->caps.virtio_queue_type & (1 << MLX5_VIRTQ_TYPE_PACKED))
  87                 *features |= (1ULL << VIRTIO_F_RING_PACKED);
  88         if (priv->caps.tso_ipv4)
  89                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO4);
  90         if (priv->caps.tso_ipv6)
  91                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO6);
  92         if (priv->caps.tx_csum)
  93                 *features |= (1ULL << VIRTIO_NET_F_CSUM);
  94         if (priv->caps.rx_csum)
  95                 *features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
  96         if (priv->caps.virtio_version_1_0)
  97                 *features |= (1ULL << VIRTIO_F_VERSION_1);
  98         return 0;
  99 }
 100
 101 static int
 102 mlx5_vdpa_get_protocol_features(int did, uint64_t *features)
 103 {
 104         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 105
 106         if (priv == NULL) {
 107                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 108                 return -1;
 109         }
 110         *features = MLX5_VDPA_PROTOCOL_FEATURES;
 111         return 0;
 112 }
 113
 114 static int
 115 mlx5_vdpa_set_vring_state(int vid, int vring, int state)
 116 {
 117         int did = rte_vhost_get_vdpa_device_id(vid);
 118         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 119         struct mlx5_vdpa_virtq *virtq = NULL;
 120
 121         if (priv == NULL) {
 122                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 123                 return -EINVAL;
 124         }
 125         SLIST_FOREACH(virtq, &priv->virtq_list, next)
 126                 if (virtq->index == vring)
 127                         break;
 128         if (!virtq) {
 129                 DRV_LOG(ERR, "Invalid or unconfigured vring id: %d.", vring);
 130                 return -EINVAL;
 131         }
 132         return mlx5_vdpa_virtq_enable(virtq, state);
 133 }
 134
 135 static int
 136 mlx5_vdpa_direct_db_prepare(struct mlx5_vdpa_priv *priv)
 137 {
 138         int ret;
 139
 140         if (priv->direct_notifier) {
 141                 ret = rte_vhost_host_notifier_ctrl(priv->vid, false);
 142                 if (ret != 0) {
 143                         DRV_LOG(INFO, "Direct HW notifier FD cannot be "
 144                                 "destroyed for device %d: %d.", priv->vid, ret);
 145                         return -1;
 146                 }
 147                 priv->direct_notifier = 0;
 148         }
 149         ret = rte_vhost_host_notifier_ctrl(priv->vid, true);
 150         if (ret != 0)
 151                 DRV_LOG(INFO, "Direct HW notifier FD cannot be configured for"
 152                         " device %d: %d.", priv->vid, ret);
 153         else
 154                 priv->direct_notifier = 1;
 155         return 0;
 156 }
 157
 158 static int
 159 mlx5_vdpa_features_set(int vid)
 160 {
 161         int did = rte_vhost_get_vdpa_device_id(vid);
 162         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 163         uint64_t log_base, log_size;
 164         uint64_t features;
 165         int ret;
 166
 167         if (priv == NULL) {
 168                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 169                 return -EINVAL;
 170         }
 171         ret = rte_vhost_get_negotiated_features(vid, &features);
 172         if (ret) {
 173                 DRV_LOG(ERR, "Failed to get negotiated features.");
 174                 return ret;
 175         }
 176         if (RTE_VHOST_NEED_LOG(features)) {
 177                 ret = rte_vhost_get_log_base(vid, &log_base, &log_size);
 178                 if (ret) {
 179                         DRV_LOG(ERR, "Failed to get log base.");
 180                         return ret;
 181                 }
 182                 ret = mlx5_vdpa_dirty_bitmap_set(priv, log_base, log_size);
 183                 if (ret) {
 184                         DRV_LOG(ERR, "Failed to set dirty bitmap.");
 185                         return ret;
 186                 }
 187                 DRV_LOG(INFO, "mlx5 vdpa: enabling dirty logging...");
 188                 ret = mlx5_vdpa_logging_enable(priv, 1);
 189                 if (ret) {
 190                         DRV_LOG(ERR, "Failed t enable dirty logging.");
 191                         return ret;
 192                 }
 193         }
 194         return 0;
 195 }
 196
 197 static int
 198 mlx5_vdpa_dev_close(int vid)
 199 {
 200         int did = rte_vhost_get_vdpa_device_id(vid);
 201         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 202         int ret = 0;
 203
 204         if (priv == NULL) {
 205                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 206                 return -1;
 207         }
 208         if (priv->configured)
 209                 ret |= mlx5_vdpa_lm_log(priv);
 210         mlx5_vdpa_cqe_event_unset(priv);
 211         ret |= mlx5_vdpa_steer_unset(priv);
 212         mlx5_vdpa_virtqs_release(priv);
 213         mlx5_vdpa_event_qp_global_release(priv);
 214         mlx5_vdpa_mem_dereg(priv);
 215         priv->configured = 0;
 216         priv->vid = 0;
 217         DRV_LOG(INFO, "vDPA device %d was closed.", vid);
 218         return ret;
 219 }
 220
 221 static int
 222 mlx5_vdpa_dev_config(int vid)
 223 {
 224         int did = rte_vhost_get_vdpa_device_id(vid);
 225         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 226
 227         if (priv == NULL) {
 228                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 229                 return -EINVAL;
 230         }
 231         if (priv->configured && mlx5_vdpa_dev_close(vid)) {
 232                 DRV_LOG(ERR, "Failed to reconfigure vid %d.", vid);
 233                 return -1;
 234         }
 235         priv->vid = vid;
 236         if (mlx5_vdpa_mem_register(priv) || mlx5_vdpa_direct_db_prepare(priv) ||
 237             mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) ||
 238             mlx5_vdpa_cqe_event_setup(priv)) {
 239                 mlx5_vdpa_dev_close(vid);
 240                 return -1;
 241         }
 242         priv->configured = 1;
 243         DRV_LOG(INFO, "vDPA device %d was configured.", vid);
 244         return 0;
 245 }
 246
 247 static int
 248 mlx5_vdpa_get_device_fd(int vid)
 249 {
 250         int did = rte_vhost_get_vdpa_device_id(vid);
 251         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 252
 253         if (priv == NULL) {
 254                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 255                 return -EINVAL;
 256         }
 257         return priv->ctx->cmd_fd;
 258 }
 259
 260 static int
 261 mlx5_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
 262 {
 263         int did = rte_vhost_get_vdpa_device_id(vid);
 264         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 265
 266         RTE_SET_USED(qid);
 267         if (priv == NULL) {
 268                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 269                 return -EINVAL;
 270         }
 271         if (!priv->var) {
 272                 DRV_LOG(ERR, "VAR was not created for device %d, is the device"
 273                         " configured?.", did);
 274                 return -EINVAL;
 275         }
 276         *offset = priv->var->mmap_off;
 277         *size = priv->var->length;
 278         return 0;
 279 }
 280
 281 static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {
 282         .get_queue_num = mlx5_vdpa_get_queue_num,
 283         .get_features = mlx5_vdpa_get_vdpa_features,
 284         .get_protocol_features = mlx5_vdpa_get_protocol_features,
 285         .dev_conf = mlx5_vdpa_dev_config,
 286         .dev_close = mlx5_vdpa_dev_close,
 287         .set_vring_state = mlx5_vdpa_set_vring_state,
 288         .set_features = mlx5_vdpa_features_set,
 289         .migration_done = NULL,
 290         .get_vfio_group_fd = NULL,
 291         .get_vfio_device_fd = mlx5_vdpa_get_device_fd,
 292         .get_notify_area = mlx5_vdpa_get_notify_area,
 293 };
 294
 295 static struct ibv_device *
 296 mlx5_vdpa_get_ib_device_match(struct rte_pci_addr *addr)
 297 {
 298         int n;
 299         struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
 300         struct ibv_device *ibv_match = NULL;
 301
 302         if (!ibv_list) {
 303                 rte_errno = ENOSYS;
 304                 return NULL;
 305         }
 306         while (n-- > 0) {
 307                 struct rte_pci_addr pci_addr;
 308
 309                 DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
 310                 if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &pci_addr))
 311                         continue;
 312                 if (memcmp(addr, &pci_addr, sizeof(pci_addr)))
 313                         continue;
 314                 ibv_match = ibv_list[n];
 315                 break;
 316         }
 317         if (!ibv_match)
 318                 rte_errno = ENOENT;
 319         mlx5_glue->free_device_list(ibv_list);
 320         return ibv_match;
 321 }
 322
 323 /* Try to disable ROCE by Netlink\Devlink. */
 324 static int
 325 mlx5_vdpa_nl_roce_disable(const char *addr)
 326 {
 327         int nlsk_fd = mlx5_nl_init(NETLINK_GENERIC);
 328         int devlink_id;
 329         int enable;
 330         int ret;
 331
 332         if (nlsk_fd < 0)
 333                 return nlsk_fd;
 334         devlink_id = mlx5_nl_devlink_family_id_get(nlsk_fd);
 335         if (devlink_id < 0) {
 336                 ret = devlink_id;
 337                 DRV_LOG(DEBUG, "Failed to get devlink id for ROCE operations by"
 338                         " Netlink.");
 339                 goto close;
 340         }
 341         ret = mlx5_nl_enable_roce_get(nlsk_fd, devlink_id, addr, &enable);
 342         if (ret) {
 343                 DRV_LOG(DEBUG, "Failed to get ROCE enable by Netlink: %d.",
 344                         ret);
 345                 goto close;
 346         } else if (!enable) {
 347                 DRV_LOG(INFO, "ROCE has already disabled(Netlink).");
 348                 goto close;
 349         }
 350         ret = mlx5_nl_enable_roce_set(nlsk_fd, devlink_id, addr, 0);
 351         if (ret)
 352                 DRV_LOG(DEBUG, "Failed to disable ROCE by Netlink: %d.", ret);
 353         else
 354                 DRV_LOG(INFO, "ROCE is disabled by Netlink successfully.");
 355 close:
 356         close(nlsk_fd);
 357         return ret;
 358 }
 359
 360 /* Try to disable ROCE by sysfs. */
 361 static int
 362 mlx5_vdpa_sys_roce_disable(const char *addr)
 363 {
 364         FILE *file_o;
 365         int enable;
 366         int ret;
 367
 368         MKSTR(file_p, "/sys/bus/pci/devices/%s/roce_enable", addr);
 369         file_o = fopen(file_p, "rb");
 370         if (!file_o) {
 371                 rte_errno = ENOTSUP;
 372                 return -ENOTSUP;
 373         }
 374         ret = fscanf(file_o, "%d", &enable);
 375         if (ret != 1) {
 376                 rte_errno = EINVAL;
 377                 ret = EINVAL;
 378                 goto close;
 379         } else if (!enable) {
 380                 ret = 0;
 381                 DRV_LOG(INFO, "ROCE has already disabled(sysfs).");
 382                 goto close;
 383         }
 384         fclose(file_o);
 385         file_o = fopen(file_p, "wb");
 386         if (!file_o) {
 387                 rte_errno = ENOTSUP;
 388                 return -ENOTSUP;
 389         }
 390         fprintf(file_o, "0\n");
 391         ret = 0;
 392 close:
 393         if (ret)
 394                 DRV_LOG(DEBUG, "Failed to disable ROCE by sysfs: %d.", ret);
 395         else
 396                 DRV_LOG(INFO, "ROCE is disabled by sysfs successfully.");
 397         fclose(file_o);
 398         return ret;
 399 }
 400
 401 #define MLX5_VDPA_MAX_RETRIES 20
 402 #define MLX5_VDPA_USEC 1000
 403 static int
 404 mlx5_vdpa_roce_disable(struct rte_pci_addr *addr, struct ibv_device **ibv)
 405 {
 406         char addr_name[64] = {0};
 407
 408         rte_pci_device_name(addr, addr_name, sizeof(addr_name));
 409         /* Firstly try to disable ROCE by Netlink and fallback to sysfs. */
 410         if (mlx5_vdpa_nl_roce_disable(addr_name) == 0 ||
 411             mlx5_vdpa_sys_roce_disable(addr_name) == 0) {
 412                 /*
 413                  * Succeed to disable ROCE, wait for the IB device to appear
 414                  * again after reload.
 415                  */
 416                 int r;
 417                 struct ibv_device *ibv_new;
 418
 419                 for (r = MLX5_VDPA_MAX_RETRIES; r; r--) {
 420                         ibv_new = mlx5_vdpa_get_ib_device_match(addr);
 421                         if (ibv_new) {
 422                                 *ibv = ibv_new;
 423                                 return 0;
 424                         }
 425                         usleep(MLX5_VDPA_USEC);
 426                 }
 427                 DRV_LOG(ERR, "Cannot much device %s after ROCE disable, "
 428                         "retries exceed %d", addr_name, MLX5_VDPA_MAX_RETRIES);
 429                 rte_errno = EAGAIN;
 430         }
 431         return -rte_errno;
 432 }
 433
 434 /**
 435  * DPDK callback to register a PCI device.
 436  *
 437  * This function spawns vdpa device out of a given PCI device.
 438  *
 439  * @param[in] pci_drv
 440  *   PCI driver structure (mlx5_vpda_driver).
 441  * @param[in] pci_dev
 442  *   PCI device information.
 443  *
 444  * @return
 445  *   0 on success, 1 to skip this driver, a negative errno value otherwise
 446  *   and rte_errno is set.
 447  */
 448 static int
 449 mlx5_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 450                     struct rte_pci_device *pci_dev __rte_unused)
 451 {
 452         struct ibv_device *ibv;
 453         struct mlx5_vdpa_priv *priv = NULL;
 454         struct ibv_context *ctx = NULL;
 455         struct mlx5_hca_attr attr;
 456         int ret;
 457
 458         if (mlx5_class_get(pci_dev->device.devargs) != MLX5_CLASS_VDPA) {
 459                 DRV_LOG(DEBUG, "Skip probing - should be probed by other mlx5"
 460                         " driver.");
 461                 return 1;
 462         }
 463         ibv = mlx5_vdpa_get_ib_device_match(&pci_dev->addr);
 464         if (!ibv) {
 465                 DRV_LOG(ERR, "No matching IB device for PCI slot "
 466                         PCI_PRI_FMT ".", pci_dev->addr.domain,
 467                         pci_dev->addr.bus, pci_dev->addr.devid,
 468                         pci_dev->addr.function);
 469                 return -rte_errno;
 470         } else {
 471                 DRV_LOG(INFO, "PCI information matches for device \"%s\".",
 472                         ibv->name);
 473         }
 474         if (mlx5_vdpa_roce_disable(&pci_dev->addr, &ibv) != 0) {
 475                 DRV_LOG(WARNING, "Failed to disable ROCE for \"%s\".",
 476                         ibv->name);
 477                 return -rte_errno;
 478         }
 479         ctx = mlx5_glue->dv_open_device(ibv);
 480         if (!ctx) {
 481                 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
 482                 rte_errno = ENODEV;
 483                 return -rte_errno;
 484         }
 485         priv = rte_zmalloc("mlx5 vDPA device private", sizeof(*priv),
 486                            RTE_CACHE_LINE_SIZE);
 487         if (!priv) {
 488                 DRV_LOG(ERR, "Failed to allocate private memory.");
 489                 rte_errno = ENOMEM;
 490                 goto error;
 491         }
 492         ret = mlx5_devx_cmd_query_hca_attr(ctx, &attr);
 493         if (ret) {
 494                 DRV_LOG(ERR, "Unable to read HCA capabilities.");
 495                 rte_errno = ENOTSUP;
 496                 goto error;
 497         } else {
 498                 if (!attr.vdpa.valid || !attr.vdpa.max_num_virtio_queues) {
 499                         DRV_LOG(ERR, "Not enough capabilities to support vdpa,"
 500                                 " maybe old FW/OFED version?");
 501                         rte_errno = ENOTSUP;
 502                         goto error;
 503                 }
 504                 priv->caps = attr.vdpa;
 505                 priv->log_max_rqt_size = attr.log_max_rqt_size;
 506         }
 507         priv->ctx = ctx;
 508         priv->dev_addr.pci_addr = pci_dev->addr;
 509         priv->dev_addr.type = VDPA_ADDR_PCI;
 510         priv->var = mlx5_glue->dv_alloc_var(ctx, 0);
 511         if (!priv->var) {
 512                 DRV_LOG(ERR, "Failed to allocate VAR %u.\n", errno);
 513                 goto error;
 514         }
 515         priv->id = rte_vdpa_register_device(&priv->dev_addr, &mlx5_vdpa_ops);
 516         if (priv->id < 0) {
 517                 DRV_LOG(ERR, "Failed to register vDPA device.");
 518                 rte_errno = rte_errno ? rte_errno : EINVAL;
 519                 goto error;
 520         }
 521         SLIST_INIT(&priv->mr_list);
 522         SLIST_INIT(&priv->virtq_list);
 523         pthread_mutex_lock(&priv_list_lock);
 524         TAILQ_INSERT_TAIL(&priv_list, priv, next);
 525         pthread_mutex_unlock(&priv_list_lock);
 526         return 0;
 527
 528 error:
 529         if (priv) {
 530                 if (priv->var)
 531                         mlx5_glue->dv_free_var(priv->var);
 532                 rte_free(priv);
 533         }
 534         if (ctx)
 535                 mlx5_glue->close_device(ctx);
 536         return -rte_errno;
 537 }
 538
 539 /**
 540  * DPDK callback to remove a PCI device.
 541  *
 542  * This function removes all vDPA devices belong to a given PCI device.
 543  *
 544  * @param[in] pci_dev
 545  *   Pointer to the PCI device.
 546  *
 547  * @return
 548  *   0 on success, the function cannot fail.
 549  */
 550 static int
 551 mlx5_vdpa_pci_remove(struct rte_pci_device *pci_dev)
 552 {
 553         struct mlx5_vdpa_priv *priv = NULL;
 554         int found = 0;
 555
 556         pthread_mutex_lock(&priv_list_lock);
 557         TAILQ_FOREACH(priv, &priv_list, next) {
 558                 if (memcmp(&priv->dev_addr.pci_addr, &pci_dev->addr,
 559                            sizeof(pci_dev->addr)) == 0) {
 560                         found = 1;
 561                         break;
 562                 }
 563         }
 564         if (found)
 565                 TAILQ_REMOVE(&priv_list, priv, next);
 566         pthread_mutex_unlock(&priv_list_lock);
 567         if (found) {
 568                 if (priv->configured)
 569                         mlx5_vdpa_dev_close(priv->vid);
 570                 if (priv->var) {
 571                         mlx5_glue->dv_free_var(priv->var);
 572                         priv->var = NULL;
 573                 }
 574                 mlx5_glue->close_device(priv->ctx);
 575                 rte_free(priv);
 576         }
 577         return 0;
 578 }
 579
 580 static const struct rte_pci_id mlx5_vdpa_pci_id_map[] = {
 581         {
 582                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 583                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 584         },
 585         {
 586                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 587                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF)
 588         },
 589         {
 590                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 591                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
 592         },
 593         {
 594                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 595                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF)
 596         },
 597         {
 598                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 599                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
 600         },
 601         {
 602                 .vendor_id = 0
 603         }
 604 };
 605
 606 static struct rte_pci_driver mlx5_vdpa_driver = {
 607         .driver = {
 608                 .name = "mlx5_vdpa",
 609         },
 610         .id_table = mlx5_vdpa_pci_id_map,
 611         .probe = mlx5_vdpa_pci_probe,
 612         .remove = mlx5_vdpa_pci_remove,
 613         .drv_flags = 0,
 614 };
 615
 616 /**
 617  * Driver initialization routine.
 618  */
 619 RTE_INIT(rte_mlx5_vdpa_init)
 620 {
 621         /* Initialize common log type. */
 622         mlx5_vdpa_logtype = rte_log_register("pmd.vdpa.mlx5");
 623         if (mlx5_vdpa_logtype >= 0)
 624                 rte_log_set_level(mlx5_vdpa_logtype, RTE_LOG_NOTICE);
 625         if (mlx5_glue)
 626                 rte_pci_register(&mlx5_vdpa_driver);
 627 }
 628
 629 RTE_PMD_EXPORT_NAME(net_mlx5_vdpa, __COUNTER__);
 630 RTE_PMD_REGISTER_PCI_TABLE(net_mlx5_vdpa, mlx5_vdpa_pci_id_map);
 631 RTE_PMD_REGISTER_KMOD_DEP(net_mlx5_vdpa, "* ib_uverbs & mlx5_core & mlx5_ib");