drivers/vdpa/mlx5/mlx5_vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <unistd.h>
   5
   6 #include <rte_malloc.h>
   7 #include <rte_log.h>
   8 #include <rte_errno.h>
   9 #include <rte_bus_pci.h>
  10 #include <rte_pci.h>
  11
  12 #include <mlx5_glue.h>
  13 #include <mlx5_common.h>
  14 #include <mlx5_devx_cmds.h>
  15 #include <mlx5_prm.h>
  16 #include <mlx5_nl.h>
  17
  18 #include "mlx5_vdpa_utils.h"
  19 #include "mlx5_vdpa.h"
  20
  21
  22 #define MLX5_VDPA_DEFAULT_FEATURES ((1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
  23                             (1ULL << VIRTIO_F_ANY_LAYOUT) | \
  24                             (1ULL << VIRTIO_NET_F_MQ) | \
  25                             (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \
  26                             (1ULL << VIRTIO_F_ORDER_PLATFORM) | \
  27                             (1ULL << VHOST_F_LOG_ALL))
  28
  29 #define MLX5_VDPA_PROTOCOL_FEATURES \
  30                             ((1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
  31                              (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
  32                              (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
  33                              (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) | \
  34                              (1ULL << VHOST_USER_PROTOCOL_F_MQ))
  35
  36 TAILQ_HEAD(mlx5_vdpa_privs, mlx5_vdpa_priv) priv_list =
  37                                               TAILQ_HEAD_INITIALIZER(priv_list);
  38 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
  39 int mlx5_vdpa_logtype;
  40
  41 static struct mlx5_vdpa_priv *
  42 mlx5_vdpa_find_priv_resource_by_did(int did)
  43 {
  44         struct mlx5_vdpa_priv *priv;
  45         int found = 0;
  46
  47         pthread_mutex_lock(&priv_list_lock);
  48         TAILQ_FOREACH(priv, &priv_list, next) {
  49                 if (did == priv->id) {
  50                         found = 1;
  51                         break;
  52                 }
  53         }
  54         pthread_mutex_unlock(&priv_list_lock);
  55         if (!found) {
  56                 DRV_LOG(ERR, "Invalid device id: %d.", did);
  57                 rte_errno = EINVAL;
  58                 return NULL;
  59         }
  60         return priv;
  61 }
  62
  63 static int
  64 mlx5_vdpa_get_queue_num(int did, uint32_t *queue_num)
  65 {
  66         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
  67
  68         if (priv == NULL) {
  69                 DRV_LOG(ERR, "Invalid device id: %d.", did);
  70                 return -1;
  71         }
  72         *queue_num = priv->caps.max_num_virtio_queues;
  73         return 0;
  74 }
  75
  76 static int
  77 mlx5_vdpa_get_vdpa_features(int did, uint64_t *features)
  78 {
  79         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
  80
  81         if (priv == NULL) {
  82                 DRV_LOG(ERR, "Invalid device id: %d.", did);
  83                 return -1;
  84         }
  85         *features = MLX5_VDPA_DEFAULT_FEATURES;
  86         if (priv->caps.virtio_queue_type & (1 << MLX5_VIRTQ_TYPE_PACKED))
  87                 *features |= (1ULL << VIRTIO_F_RING_PACKED);
  88         if (priv->caps.tso_ipv4)
  89                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO4);
  90         if (priv->caps.tso_ipv6)
  91                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO6);
  92         if (priv->caps.tx_csum)
  93                 *features |= (1ULL << VIRTIO_NET_F_CSUM);
  94         if (priv->caps.rx_csum)
  95                 *features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
  96         if (priv->caps.virtio_version_1_0)
  97                 *features |= (1ULL << VIRTIO_F_VERSION_1);
  98         return 0;
  99 }
 100
 101 static int
 102 mlx5_vdpa_get_protocol_features(int did, uint64_t *features)
 103 {
 104         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 105
 106         if (priv == NULL) {
 107                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 108                 return -1;
 109         }
 110         *features = MLX5_VDPA_PROTOCOL_FEATURES;
 111         return 0;
 112 }
 113
 114 static int
 115 mlx5_vdpa_set_vring_state(int vid, int vring, int state)
 116 {
 117         int did = rte_vhost_get_vdpa_device_id(vid);
 118         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 119         struct mlx5_vdpa_virtq *virtq = NULL;
 120
 121         if (priv == NULL) {
 122                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 123                 return -EINVAL;
 124         }
 125         SLIST_FOREACH(virtq, &priv->virtq_list, next)
 126                 if (virtq->index == vring)
 127                         break;
 128         if (!virtq) {
 129                 DRV_LOG(ERR, "Invalid or unconfigured vring id: %d.", vring);
 130                 return -EINVAL;
 131         }
 132         return mlx5_vdpa_virtq_enable(virtq, state);
 133 }
 134
 135 static int
 136 mlx5_vdpa_direct_db_prepare(struct mlx5_vdpa_priv *priv)
 137 {
 138         int ret;
 139
 140         if (priv->direct_notifier) {
 141                 ret = rte_vhost_host_notifier_ctrl(priv->vid, false);
 142                 if (ret != 0) {
 143                         DRV_LOG(INFO, "Direct HW notifier FD cannot be "
 144                                 "destroyed for device %d: %d.", priv->vid, ret);
 145                         return -1;
 146                 }
 147                 priv->direct_notifier = 0;
 148         }
 149         ret = rte_vhost_host_notifier_ctrl(priv->vid, true);
 150         if (ret != 0)
 151                 DRV_LOG(INFO, "Direct HW notifier FD cannot be configured for"
 152                         " device %d: %d.", priv->vid, ret);
 153         else
 154                 priv->direct_notifier = 1;
 155         return 0;
 156 }
 157
 158 static int
 159 mlx5_vdpa_features_set(int vid)
 160 {
 161         int did = rte_vhost_get_vdpa_device_id(vid);
 162         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 163         uint64_t log_base, log_size;
 164         uint64_t features;
 165         int ret;
 166
 167         if (priv == NULL) {
 168                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 169                 return -EINVAL;
 170         }
 171         ret = rte_vhost_get_negotiated_features(vid, &features);
 172         if (ret) {
 173                 DRV_LOG(ERR, "Failed to get negotiated features.");
 174                 return ret;
 175         }
 176         if (RTE_VHOST_NEED_LOG(features)) {
 177                 ret = rte_vhost_get_log_base(vid, &log_base, &log_size);
 178                 if (ret) {
 179                         DRV_LOG(ERR, "Failed to get log base.");
 180                         return ret;
 181                 }
 182                 ret = mlx5_vdpa_dirty_bitmap_set(priv, log_base, log_size);
 183                 if (ret) {
 184                         DRV_LOG(ERR, "Failed to set dirty bitmap.");
 185                         return ret;
 186                 }
 187                 DRV_LOG(INFO, "mlx5 vdpa: enabling dirty logging...");
 188                 ret = mlx5_vdpa_logging_enable(priv, 1);
 189                 if (ret) {
 190                         DRV_LOG(ERR, "Failed t enable dirty logging.");
 191                         return ret;
 192                 }
 193         }
 194         return 0;
 195 }
 196
 197 static int
 198 mlx5_vdpa_dev_close(int vid)
 199 {
 200         int did = rte_vhost_get_vdpa_device_id(vid);
 201         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 202         int ret = 0;
 203
 204         if (priv == NULL) {
 205                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 206                 return -1;
 207         }
 208         if (priv->configured)
 209                 ret |= mlx5_vdpa_lm_log(priv);
 210         mlx5_vdpa_cqe_event_unset(priv);
 211         ret |= mlx5_vdpa_steer_unset(priv);
 212         mlx5_vdpa_virtqs_release(priv);
 213         mlx5_vdpa_event_qp_global_release(priv);
 214         mlx5_vdpa_mem_dereg(priv);
 215         priv->configured = 0;
 216         priv->vid = 0;
 217         return ret;
 218 }
 219
 220 static int
 221 mlx5_vdpa_dev_config(int vid)
 222 {
 223         int did = rte_vhost_get_vdpa_device_id(vid);
 224         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 225
 226         if (priv == NULL) {
 227                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 228                 return -EINVAL;
 229         }
 230         if (priv->configured && mlx5_vdpa_dev_close(vid)) {
 231                 DRV_LOG(ERR, "Failed to reconfigure vid %d.", vid);
 232                 return -1;
 233         }
 234         priv->vid = vid;
 235         if (mlx5_vdpa_mem_register(priv) || mlx5_vdpa_direct_db_prepare(priv) ||
 236             mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) ||
 237             mlx5_vdpa_cqe_event_setup(priv)) {
 238                 mlx5_vdpa_dev_close(vid);
 239                 return -1;
 240         }
 241         priv->configured = 1;
 242         return 0;
 243 }
 244
 245 static int
 246 mlx5_vdpa_get_device_fd(int vid)
 247 {
 248         int did = rte_vhost_get_vdpa_device_id(vid);
 249         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 250
 251         if (priv == NULL) {
 252                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 253                 return -EINVAL;
 254         }
 255         return priv->ctx->cmd_fd;
 256 }
 257
 258 static int
 259 mlx5_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
 260 {
 261         int did = rte_vhost_get_vdpa_device_id(vid);
 262         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 263
 264         RTE_SET_USED(qid);
 265         if (priv == NULL) {
 266                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 267                 return -EINVAL;
 268         }
 269         if (!priv->var) {
 270                 DRV_LOG(ERR, "VAR was not created for device %d, is the device"
 271                         " configured?.", did);
 272                 return -EINVAL;
 273         }
 274         *offset = priv->var->mmap_off;
 275         *size = priv->var->length;
 276         return 0;
 277 }
 278
 279 static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {
 280         .get_queue_num = mlx5_vdpa_get_queue_num,
 281         .get_features = mlx5_vdpa_get_vdpa_features,
 282         .get_protocol_features = mlx5_vdpa_get_protocol_features,
 283         .dev_conf = mlx5_vdpa_dev_config,
 284         .dev_close = mlx5_vdpa_dev_close,
 285         .set_vring_state = mlx5_vdpa_set_vring_state,
 286         .set_features = mlx5_vdpa_features_set,
 287         .migration_done = NULL,
 288         .get_vfio_group_fd = NULL,
 289         .get_vfio_device_fd = mlx5_vdpa_get_device_fd,
 290         .get_notify_area = mlx5_vdpa_get_notify_area,
 291 };
 292
 293 static struct ibv_device *
 294 mlx5_vdpa_get_ib_device_match(struct rte_pci_addr *addr)
 295 {
 296         int n;
 297         struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
 298         struct ibv_device *ibv_match = NULL;
 299
 300         if (!ibv_list) {
 301                 rte_errno = ENOSYS;
 302                 return NULL;
 303         }
 304         while (n-- > 0) {
 305                 struct rte_pci_addr pci_addr;
 306
 307                 DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
 308                 if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &pci_addr))
 309                         continue;
 310                 if (memcmp(addr, &pci_addr, sizeof(pci_addr)))
 311                         continue;
 312                 ibv_match = ibv_list[n];
 313                 break;
 314         }
 315         if (!ibv_match)
 316                 rte_errno = ENOENT;
 317         mlx5_glue->free_device_list(ibv_list);
 318         return ibv_match;
 319 }
 320
 321 /* Try to disable ROCE by Netlink\Devlink. */
 322 static int
 323 mlx5_vdpa_nl_roce_disable(const char *addr)
 324 {
 325         int nlsk_fd = mlx5_nl_init(NETLINK_GENERIC);
 326         int devlink_id;
 327         int enable;
 328         int ret;
 329
 330         if (nlsk_fd < 0)
 331                 return nlsk_fd;
 332         devlink_id = mlx5_nl_devlink_family_id_get(nlsk_fd);
 333         if (devlink_id < 0) {
 334                 ret = devlink_id;
 335                 DRV_LOG(DEBUG, "Failed to get devlink id for ROCE operations by"
 336                         " Netlink.");
 337                 goto close;
 338         }
 339         ret = mlx5_nl_enable_roce_get(nlsk_fd, devlink_id, addr, &enable);
 340         if (ret) {
 341                 DRV_LOG(DEBUG, "Failed to get ROCE enable by Netlink: %d.",
 342                         ret);
 343                 goto close;
 344         } else if (!enable) {
 345                 DRV_LOG(INFO, "ROCE has already disabled(Netlink).");
 346                 goto close;
 347         }
 348         ret = mlx5_nl_enable_roce_set(nlsk_fd, devlink_id, addr, 0);
 349         if (ret)
 350                 DRV_LOG(DEBUG, "Failed to disable ROCE by Netlink: %d.", ret);
 351         else
 352                 DRV_LOG(INFO, "ROCE is disabled by Netlink successfully.");
 353 close:
 354         close(nlsk_fd);
 355         return ret;
 356 }
 357
 358 /* Try to disable ROCE by sysfs. */
 359 static int
 360 mlx5_vdpa_sys_roce_disable(const char *addr)
 361 {
 362         FILE *file_o;
 363         int enable;
 364         int ret;
 365
 366         MKSTR(file_p, "/sys/bus/pci/devices/%s/roce_enable", addr);
 367         file_o = fopen(file_p, "rb");
 368         if (!file_o) {
 369                 rte_errno = ENOTSUP;
 370                 return -ENOTSUP;
 371         }
 372         ret = fscanf(file_o, "%d", &enable);
 373         if (ret != 1) {
 374                 rte_errno = EINVAL;
 375                 ret = EINVAL;
 376                 goto close;
 377         } else if (!enable) {
 378                 ret = 0;
 379                 DRV_LOG(INFO, "ROCE has already disabled(sysfs).");
 380                 goto close;
 381         }
 382         fclose(file_o);
 383         file_o = fopen(file_p, "wb");
 384         if (!file_o) {
 385                 rte_errno = ENOTSUP;
 386                 return -ENOTSUP;
 387         }
 388         fprintf(file_o, "0\n");
 389         ret = 0;
 390 close:
 391         if (ret)
 392                 DRV_LOG(DEBUG, "Failed to disable ROCE by sysfs: %d.", ret);
 393         else
 394                 DRV_LOG(INFO, "ROCE is disabled by sysfs successfully.");
 395         fclose(file_o);
 396         return ret;
 397 }
 398
 399 #define MLX5_VDPA_MAX_RETRIES 20
 400 #define MLX5_VDPA_USEC 1000
 401 static int
 402 mlx5_vdpa_roce_disable(struct rte_pci_addr *addr, struct ibv_device **ibv)
 403 {
 404         char addr_name[64] = {0};
 405
 406         rte_pci_device_name(addr, addr_name, sizeof(addr_name));
 407         /* Firstly try to disable ROCE by Netlink and fallback to sysfs. */
 408         if (mlx5_vdpa_nl_roce_disable(addr_name) == 0 ||
 409             mlx5_vdpa_sys_roce_disable(addr_name) == 0) {
 410                 /*
 411                  * Succeed to disable ROCE, wait for the IB device to appear
 412                  * again after reload.
 413                  */
 414                 int r;
 415                 struct ibv_device *ibv_new;
 416
 417                 for (r = MLX5_VDPA_MAX_RETRIES; r; r--) {
 418                         ibv_new = mlx5_vdpa_get_ib_device_match(addr);
 419                         if (ibv_new) {
 420                                 *ibv = ibv_new;
 421                                 return 0;
 422                         }
 423                         usleep(MLX5_VDPA_USEC);
 424                 }
 425                 DRV_LOG(ERR, "Cannot much device %s after ROCE disable, "
 426                         "retries exceed %d", addr_name, MLX5_VDPA_MAX_RETRIES);
 427                 rte_errno = EAGAIN;
 428         }
 429         return -rte_errno;
 430 }
 431
 432 /**
 433  * DPDK callback to register a PCI device.
 434  *
 435  * This function spawns vdpa device out of a given PCI device.
 436  *
 437  * @param[in] pci_drv
 438  *   PCI driver structure (mlx5_vpda_driver).
 439  * @param[in] pci_dev
 440  *   PCI device information.
 441  *
 442  * @return
 443  *   0 on success, 1 to skip this driver, a negative errno value otherwise
 444  *   and rte_errno is set.
 445  */
 446 static int
 447 mlx5_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 448                     struct rte_pci_device *pci_dev __rte_unused)
 449 {
 450         struct ibv_device *ibv;
 451         struct mlx5_vdpa_priv *priv = NULL;
 452         struct ibv_context *ctx = NULL;
 453         struct mlx5_hca_attr attr;
 454         int ret;
 455
 456         if (mlx5_class_get(pci_dev->device.devargs) != MLX5_CLASS_VDPA) {
 457                 DRV_LOG(DEBUG, "Skip probing - should be probed by other mlx5"
 458                         " driver.");
 459                 return 1;
 460         }
 461         ibv = mlx5_vdpa_get_ib_device_match(&pci_dev->addr);
 462         if (!ibv) {
 463                 DRV_LOG(ERR, "No matching IB device for PCI slot "
 464                         PCI_PRI_FMT ".", pci_dev->addr.domain,
 465                         pci_dev->addr.bus, pci_dev->addr.devid,
 466                         pci_dev->addr.function);
 467                 return -rte_errno;
 468         } else {
 469                 DRV_LOG(INFO, "PCI information matches for device \"%s\".",
 470                         ibv->name);
 471         }
 472         if (mlx5_vdpa_roce_disable(&pci_dev->addr, &ibv) != 0) {
 473                 DRV_LOG(WARNING, "Failed to disable ROCE for \"%s\".",
 474                         ibv->name);
 475                 return -rte_errno;
 476         }
 477         ctx = mlx5_glue->dv_open_device(ibv);
 478         if (!ctx) {
 479                 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
 480                 rte_errno = ENODEV;
 481                 return -rte_errno;
 482         }
 483         priv = rte_zmalloc("mlx5 vDPA device private", sizeof(*priv),
 484                            RTE_CACHE_LINE_SIZE);
 485         if (!priv) {
 486                 DRV_LOG(ERR, "Failed to allocate private memory.");
 487                 rte_errno = ENOMEM;
 488                 goto error;
 489         }
 490         ret = mlx5_devx_cmd_query_hca_attr(ctx, &attr);
 491         if (ret) {
 492                 DRV_LOG(ERR, "Unable to read HCA capabilities.");
 493                 rte_errno = ENOTSUP;
 494                 goto error;
 495         } else {
 496                 if (!attr.vdpa.valid || !attr.vdpa.max_num_virtio_queues) {
 497                         DRV_LOG(ERR, "Not enough capabilities to support vdpa,"
 498                                 " maybe old FW/OFED version?");
 499                         rte_errno = ENOTSUP;
 500                         goto error;
 501                 }
 502                 priv->caps = attr.vdpa;
 503                 priv->log_max_rqt_size = attr.log_max_rqt_size;
 504         }
 505         priv->ctx = ctx;
 506         priv->dev_addr.pci_addr = pci_dev->addr;
 507         priv->dev_addr.type = VDPA_ADDR_PCI;
 508         priv->var = mlx5_glue->dv_alloc_var(ctx, 0);
 509         if (!priv->var) {
 510                 DRV_LOG(ERR, "Failed to allocate VAR %u.\n", errno);
 511                 goto error;
 512         }
 513         priv->id = rte_vdpa_register_device(&priv->dev_addr, &mlx5_vdpa_ops);
 514         if (priv->id < 0) {
 515                 DRV_LOG(ERR, "Failed to register vDPA device.");
 516                 rte_errno = rte_errno ? rte_errno : EINVAL;
 517                 goto error;
 518         }
 519         SLIST_INIT(&priv->mr_list);
 520         SLIST_INIT(&priv->virtq_list);
 521         pthread_mutex_lock(&priv_list_lock);
 522         TAILQ_INSERT_TAIL(&priv_list, priv, next);
 523         pthread_mutex_unlock(&priv_list_lock);
 524         return 0;
 525
 526 error:
 527         if (priv) {
 528                 if (priv->var)
 529                         mlx5_glue->dv_free_var(priv->var);
 530                 rte_free(priv);
 531         }
 532         if (ctx)
 533                 mlx5_glue->close_device(ctx);
 534         return -rte_errno;
 535 }
 536
 537 /**
 538  * DPDK callback to remove a PCI device.
 539  *
 540  * This function removes all vDPA devices belong to a given PCI device.
 541  *
 542  * @param[in] pci_dev
 543  *   Pointer to the PCI device.
 544  *
 545  * @return
 546  *   0 on success, the function cannot fail.
 547  */
 548 static int
 549 mlx5_vdpa_pci_remove(struct rte_pci_device *pci_dev)
 550 {
 551         struct mlx5_vdpa_priv *priv = NULL;
 552         int found = 0;
 553
 554         pthread_mutex_lock(&priv_list_lock);
 555         TAILQ_FOREACH(priv, &priv_list, next) {
 556                 if (memcmp(&priv->dev_addr.pci_addr, &pci_dev->addr,
 557                            sizeof(pci_dev->addr)) == 0) {
 558                         found = 1;
 559                         break;
 560                 }
 561         }
 562         if (found)
 563                 TAILQ_REMOVE(&priv_list, priv, next);
 564         pthread_mutex_unlock(&priv_list_lock);
 565         if (found) {
 566                 if (priv->configured)
 567                         mlx5_vdpa_dev_close(priv->vid);
 568                 if (priv->var) {
 569                         mlx5_glue->dv_free_var(priv->var);
 570                         priv->var = NULL;
 571                 }
 572                 mlx5_glue->close_device(priv->ctx);
 573                 rte_free(priv);
 574         }
 575         return 0;
 576 }
 577
 578 static const struct rte_pci_id mlx5_vdpa_pci_id_map[] = {
 579         {
 580                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 581                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 582         },
 583         {
 584                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 585                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF)
 586         },
 587         {
 588                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 589                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
 590         },
 591         {
 592                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 593                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF)
 594         },
 595         {
 596                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 597                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
 598         },
 599         {
 600                 .vendor_id = 0
 601         }
 602 };
 603
 604 static struct rte_pci_driver mlx5_vdpa_driver = {
 605         .driver = {
 606                 .name = "mlx5_vdpa",
 607         },
 608         .id_table = mlx5_vdpa_pci_id_map,
 609         .probe = mlx5_vdpa_pci_probe,
 610         .remove = mlx5_vdpa_pci_remove,
 611         .drv_flags = 0,
 612 };
 613
 614 /**
 615  * Driver initialization routine.
 616  */
 617 RTE_INIT(rte_mlx5_vdpa_init)
 618 {
 619         /* Initialize common log type. */
 620         mlx5_vdpa_logtype = rte_log_register("pmd.vdpa.mlx5");
 621         if (mlx5_vdpa_logtype >= 0)
 622                 rte_log_set_level(mlx5_vdpa_logtype, RTE_LOG_NOTICE);
 623         if (mlx5_glue)
 624                 rte_pci_register(&mlx5_vdpa_driver);
 625 }
 626
 627 RTE_PMD_EXPORT_NAME(net_mlx5_vdpa, __COUNTER__);
 628 RTE_PMD_REGISTER_PCI_TABLE(net_mlx5_vdpa, mlx5_vdpa_pci_id_map);
 629 RTE_PMD_REGISTER_KMOD_DEP(net_mlx5_vdpa, "* ib_uverbs & mlx5_core & mlx5_ib");