drivers/vdpa/mlx5/mlx5_vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <unistd.h>
   5
   6 #include <rte_malloc.h>
   7 #include <rte_log.h>
   8 #include <rte_errno.h>
   9 #include <rte_bus_pci.h>
  10 #include <rte_pci.h>
  11 #include <rte_string_fns.h>
  12
  13 #include <mlx5_glue.h>
  14 #include <mlx5_common.h>
  15 #include <mlx5_devx_cmds.h>
  16 #include <mlx5_prm.h>
  17 #include <mlx5_nl.h>
  18
  19 #include "mlx5_vdpa_utils.h"
  20 #include "mlx5_vdpa.h"
  21
  22
  23 #define MLX5_VDPA_DEFAULT_FEATURES ((1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
  24                             (1ULL << VIRTIO_F_ANY_LAYOUT) | \
  25                             (1ULL << VIRTIO_NET_F_MQ) | \
  26                             (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \
  27                             (1ULL << VIRTIO_F_ORDER_PLATFORM) | \
  28                             (1ULL << VHOST_F_LOG_ALL))
  29
  30 #define MLX5_VDPA_PROTOCOL_FEATURES \
  31                             ((1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
  32                              (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
  33                              (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
  34                              (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) | \
  35                              (1ULL << VHOST_USER_PROTOCOL_F_MQ))
  36
  37 TAILQ_HEAD(mlx5_vdpa_privs, mlx5_vdpa_priv) priv_list =
  38                                               TAILQ_HEAD_INITIALIZER(priv_list);
  39 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
  40 int mlx5_vdpa_logtype;
  41
  42 static struct mlx5_vdpa_priv *
  43 mlx5_vdpa_find_priv_resource_by_did(int did)
  44 {
  45         struct mlx5_vdpa_priv *priv;
  46         int found = 0;
  47
  48         pthread_mutex_lock(&priv_list_lock);
  49         TAILQ_FOREACH(priv, &priv_list, next) {
  50                 if (did == priv->id) {
  51                         found = 1;
  52                         break;
  53                 }
  54         }
  55         pthread_mutex_unlock(&priv_list_lock);
  56         if (!found) {
  57                 DRV_LOG(ERR, "Invalid device id: %d.", did);
  58                 rte_errno = EINVAL;
  59                 return NULL;
  60         }
  61         return priv;
  62 }
  63
  64 static int
  65 mlx5_vdpa_get_queue_num(int did, uint32_t *queue_num)
  66 {
  67         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
  68
  69         if (priv == NULL) {
  70                 DRV_LOG(ERR, "Invalid device id: %d.", did);
  71                 return -1;
  72         }
  73         *queue_num = priv->caps.max_num_virtio_queues;
  74         return 0;
  75 }
  76
  77 static int
  78 mlx5_vdpa_get_vdpa_features(int did, uint64_t *features)
  79 {
  80         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
  81
  82         if (priv == NULL) {
  83                 DRV_LOG(ERR, "Invalid device id: %d.", did);
  84                 return -1;
  85         }
  86         *features = MLX5_VDPA_DEFAULT_FEATURES;
  87         if (priv->caps.virtio_queue_type & (1 << MLX5_VIRTQ_TYPE_PACKED))
  88                 *features |= (1ULL << VIRTIO_F_RING_PACKED);
  89         if (priv->caps.tso_ipv4)
  90                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO4);
  91         if (priv->caps.tso_ipv6)
  92                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO6);
  93         if (priv->caps.tx_csum)
  94                 *features |= (1ULL << VIRTIO_NET_F_CSUM);
  95         if (priv->caps.rx_csum)
  96                 *features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
  97         if (priv->caps.virtio_version_1_0)
  98                 *features |= (1ULL << VIRTIO_F_VERSION_1);
  99         return 0;
 100 }
 101
 102 static int
 103 mlx5_vdpa_get_protocol_features(int did, uint64_t *features)
 104 {
 105         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 106
 107         if (priv == NULL) {
 108                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 109                 return -1;
 110         }
 111         *features = MLX5_VDPA_PROTOCOL_FEATURES;
 112         return 0;
 113 }
 114
 115 static int
 116 mlx5_vdpa_set_vring_state(int vid, int vring, int state)
 117 {
 118         int did = rte_vhost_get_vdpa_device_id(vid);
 119         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 120
 121         if (priv == NULL) {
 122                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 123                 return -EINVAL;
 124         }
 125         if (vring >= (int)priv->caps.max_num_virtio_queues * 2) {
 126                 DRV_LOG(ERR, "Too big vring id: %d.", vring);
 127                 return -E2BIG;
 128         }
 129         return mlx5_vdpa_virtq_enable(priv, vring, state);
 130 }
 131
 132 static int
 133 mlx5_vdpa_direct_db_prepare(struct mlx5_vdpa_priv *priv)
 134 {
 135         int ret;
 136
 137         if (priv->direct_notifier) {
 138                 ret = rte_vhost_host_notifier_ctrl(priv->vid, false);
 139                 if (ret != 0) {
 140                         DRV_LOG(INFO, "Direct HW notifier FD cannot be "
 141                                 "destroyed for device %d: %d.", priv->vid, ret);
 142                         return -1;
 143                 }
 144                 priv->direct_notifier = 0;
 145         }
 146         ret = rte_vhost_host_notifier_ctrl(priv->vid, true);
 147         if (ret != 0)
 148                 DRV_LOG(INFO, "Direct HW notifier FD cannot be configured for"
 149                         " device %d: %d.", priv->vid, ret);
 150         else
 151                 priv->direct_notifier = 1;
 152         return 0;
 153 }
 154
 155 static int
 156 mlx5_vdpa_features_set(int vid)
 157 {
 158         int did = rte_vhost_get_vdpa_device_id(vid);
 159         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 160         uint64_t log_base, log_size;
 161         uint64_t features;
 162         int ret;
 163
 164         if (priv == NULL) {
 165                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 166                 return -EINVAL;
 167         }
 168         ret = rte_vhost_get_negotiated_features(vid, &features);
 169         if (ret) {
 170                 DRV_LOG(ERR, "Failed to get negotiated features.");
 171                 return ret;
 172         }
 173         if (RTE_VHOST_NEED_LOG(features)) {
 174                 ret = rte_vhost_get_log_base(vid, &log_base, &log_size);
 175                 if (ret) {
 176                         DRV_LOG(ERR, "Failed to get log base.");
 177                         return ret;
 178                 }
 179                 ret = mlx5_vdpa_dirty_bitmap_set(priv, log_base, log_size);
 180                 if (ret) {
 181                         DRV_LOG(ERR, "Failed to set dirty bitmap.");
 182                         return ret;
 183                 }
 184                 DRV_LOG(INFO, "mlx5 vdpa: enabling dirty logging...");
 185                 ret = mlx5_vdpa_logging_enable(priv, 1);
 186                 if (ret) {
 187                         DRV_LOG(ERR, "Failed t enable dirty logging.");
 188                         return ret;
 189                 }
 190         }
 191         return 0;
 192 }
 193
 194 static int
 195 mlx5_vdpa_pd_create(struct mlx5_vdpa_priv *priv)
 196 {
 197 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 198         priv->pd = mlx5_glue->alloc_pd(priv->ctx);
 199         if (priv->pd == NULL) {
 200                 DRV_LOG(ERR, "Failed to allocate PD.");
 201                 return errno ? -errno : -ENOMEM;
 202         }
 203         struct mlx5dv_obj obj;
 204         struct mlx5dv_pd pd_info;
 205         int ret = 0;
 206
 207         obj.pd.in = priv->pd;
 208         obj.pd.out = &pd_info;
 209         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
 210         if (ret) {
 211                 DRV_LOG(ERR, "Fail to get PD object info.");
 212                 mlx5_glue->dealloc_pd(priv->pd);
 213                 priv->pd = NULL;
 214                 return -errno;
 215         }
 216         priv->pdn = pd_info.pdn;
 217         return 0;
 218 #else
 219         (void)priv;
 220         DRV_LOG(ERR, "Cannot get pdn - no DV support.");
 221         return -ENOTSUP;
 222 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
 223 }
 224
 225 static int
 226 mlx5_vdpa_dev_close(int vid)
 227 {
 228         int did = rte_vhost_get_vdpa_device_id(vid);
 229         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 230         int ret = 0;
 231
 232         if (priv == NULL) {
 233                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 234                 return -1;
 235         }
 236         if (priv->configured)
 237                 ret |= mlx5_vdpa_lm_log(priv);
 238         mlx5_vdpa_cqe_event_unset(priv);
 239         mlx5_vdpa_steer_unset(priv);
 240         mlx5_vdpa_virtqs_release(priv);
 241         mlx5_vdpa_event_qp_global_release(priv);
 242         mlx5_vdpa_mem_dereg(priv);
 243         if (priv->pd) {
 244                 claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 245                 priv->pd = NULL;
 246         }
 247         priv->configured = 0;
 248         priv->vid = 0;
 249         DRV_LOG(INFO, "vDPA device %d was closed.", vid);
 250         return ret;
 251 }
 252
 253 static int
 254 mlx5_vdpa_dev_config(int vid)
 255 {
 256         int did = rte_vhost_get_vdpa_device_id(vid);
 257         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 258
 259         if (priv == NULL) {
 260                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 261                 return -EINVAL;
 262         }
 263         if (priv->configured && mlx5_vdpa_dev_close(vid)) {
 264                 DRV_LOG(ERR, "Failed to reconfigure vid %d.", vid);
 265                 return -1;
 266         }
 267         priv->vid = vid;
 268         if (mlx5_vdpa_pd_create(priv) || mlx5_vdpa_mem_register(priv) ||
 269             mlx5_vdpa_direct_db_prepare(priv) ||
 270             mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) ||
 271             mlx5_vdpa_cqe_event_setup(priv)) {
 272                 mlx5_vdpa_dev_close(vid);
 273                 return -1;
 274         }
 275         priv->configured = 1;
 276         DRV_LOG(INFO, "vDPA device %d was configured.", vid);
 277         return 0;
 278 }
 279
 280 static int
 281 mlx5_vdpa_get_device_fd(int vid)
 282 {
 283         int did = rte_vhost_get_vdpa_device_id(vid);
 284         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 285
 286         if (priv == NULL) {
 287                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 288                 return -EINVAL;
 289         }
 290         return priv->ctx->cmd_fd;
 291 }
 292
 293 static int
 294 mlx5_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
 295 {
 296         int did = rte_vhost_get_vdpa_device_id(vid);
 297         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 298
 299         RTE_SET_USED(qid);
 300         if (priv == NULL) {
 301                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 302                 return -EINVAL;
 303         }
 304         if (!priv->var) {
 305                 DRV_LOG(ERR, "VAR was not created for device %d, is the device"
 306                         " configured?.", did);
 307                 return -EINVAL;
 308         }
 309         *offset = priv->var->mmap_off;
 310         *size = priv->var->length;
 311         return 0;
 312 }
 313
 314 static int
 315 mlx5_vdpa_get_stats_names(int did, struct rte_vdpa_stat_name *stats_names,
 316                           unsigned int size)
 317 {
 318         static const char *mlx5_vdpa_stats_names[MLX5_VDPA_STATS_MAX] = {
 319                 "received_descriptors",
 320                 "completed_descriptors",
 321                 "bad descriptor errors",
 322                 "exceed max chain",
 323                 "invalid buffer",
 324                 "completion errors",
 325         };
 326         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 327         unsigned int i;
 328
 329         if (priv == NULL) {
 330                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 331                 return -ENODEV;
 332         }
 333         if (!stats_names)
 334                 return MLX5_VDPA_STATS_MAX;
 335         size = RTE_MIN(size, (unsigned int)MLX5_VDPA_STATS_MAX);
 336         for (i = 0; i < size; ++i)
 337                 strlcpy(stats_names[i].name, mlx5_vdpa_stats_names[i],
 338                         RTE_VDPA_STATS_NAME_SIZE);
 339         return size;
 340 }
 341
 342 static int
 343 mlx5_vdpa_get_stats(int did, int qid, struct rte_vdpa_stat *stats,
 344                     unsigned int n)
 345 {
 346         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 347
 348         if (priv == NULL) {
 349                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 350                 return -ENODEV;
 351         }
 352         if (!priv->configured) {
 353                 DRV_LOG(ERR, "Device %d was not configured.", did);
 354                 return -ENODATA;
 355         }
 356         if (qid >= (int)priv->nr_virtqs) {
 357                 DRV_LOG(ERR, "Too big vring id: %d.", qid);
 358                 return -E2BIG;
 359         }
 360         if (!priv->caps.queue_counters_valid) {
 361                 DRV_LOG(ERR, "Virtq statistics is not supported for device %d.",
 362                         did);
 363                 return -ENOTSUP;
 364         }
 365         return mlx5_vdpa_virtq_stats_get(priv, qid, stats, n);
 366 }
 367
 368 static int
 369 mlx5_vdpa_reset_stats(int did, int qid)
 370 {
 371         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 372
 373         if (priv == NULL) {
 374                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 375                 return -ENODEV;
 376         }
 377         if (!priv->configured) {
 378                 DRV_LOG(ERR, "Device %d was not configured.", did);
 379                 return -ENODATA;
 380         }
 381         if (qid >= (int)priv->nr_virtqs) {
 382                 DRV_LOG(ERR, "Too big vring id: %d.", qid);
 383                 return -E2BIG;
 384         }
 385         if (!priv->caps.queue_counters_valid) {
 386                 DRV_LOG(ERR, "Virtq statistics is not supported for device %d.",
 387                         did);
 388                 return -ENOTSUP;
 389         }
 390         return mlx5_vdpa_virtq_stats_reset(priv, qid);
 391 }
 392
 393 static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {
 394         .get_queue_num = mlx5_vdpa_get_queue_num,
 395         .get_features = mlx5_vdpa_get_vdpa_features,
 396         .get_protocol_features = mlx5_vdpa_get_protocol_features,
 397         .dev_conf = mlx5_vdpa_dev_config,
 398         .dev_close = mlx5_vdpa_dev_close,
 399         .set_vring_state = mlx5_vdpa_set_vring_state,
 400         .set_features = mlx5_vdpa_features_set,
 401         .migration_done = NULL,
 402         .get_vfio_group_fd = NULL,
 403         .get_vfio_device_fd = mlx5_vdpa_get_device_fd,
 404         .get_notify_area = mlx5_vdpa_get_notify_area,
 405         .get_stats_names = mlx5_vdpa_get_stats_names,
 406         .get_stats = mlx5_vdpa_get_stats,
 407         .reset_stats = mlx5_vdpa_reset_stats,
 408 };
 409
 410 static struct ibv_device *
 411 mlx5_vdpa_get_ib_device_match(struct rte_pci_addr *addr)
 412 {
 413         int n;
 414         struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
 415         struct ibv_device *ibv_match = NULL;
 416
 417         if (!ibv_list) {
 418                 rte_errno = ENOSYS;
 419                 return NULL;
 420         }
 421         while (n-- > 0) {
 422                 struct rte_pci_addr pci_addr;
 423
 424                 DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
 425                 if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &pci_addr))
 426                         continue;
 427                 if (rte_pci_addr_cmp(addr, &pci_addr))
 428                         continue;
 429                 ibv_match = ibv_list[n];
 430                 break;
 431         }
 432         if (!ibv_match)
 433                 rte_errno = ENOENT;
 434         mlx5_glue->free_device_list(ibv_list);
 435         return ibv_match;
 436 }
 437
 438 /* Try to disable ROCE by Netlink\Devlink. */
 439 static int
 440 mlx5_vdpa_nl_roce_disable(const char *addr)
 441 {
 442         int nlsk_fd = mlx5_nl_init(NETLINK_GENERIC);
 443         int devlink_id;
 444         int enable;
 445         int ret;
 446
 447         if (nlsk_fd < 0)
 448                 return nlsk_fd;
 449         devlink_id = mlx5_nl_devlink_family_id_get(nlsk_fd);
 450         if (devlink_id < 0) {
 451                 ret = devlink_id;
 452                 DRV_LOG(DEBUG, "Failed to get devlink id for ROCE operations by"
 453                         " Netlink.");
 454                 goto close;
 455         }
 456         ret = mlx5_nl_enable_roce_get(nlsk_fd, devlink_id, addr, &enable);
 457         if (ret) {
 458                 DRV_LOG(DEBUG, "Failed to get ROCE enable by Netlink: %d.",
 459                         ret);
 460                 goto close;
 461         } else if (!enable) {
 462                 DRV_LOG(INFO, "ROCE has already disabled(Netlink).");
 463                 goto close;
 464         }
 465         ret = mlx5_nl_enable_roce_set(nlsk_fd, devlink_id, addr, 0);
 466         if (ret)
 467                 DRV_LOG(DEBUG, "Failed to disable ROCE by Netlink: %d.", ret);
 468         else
 469                 DRV_LOG(INFO, "ROCE is disabled by Netlink successfully.");
 470 close:
 471         close(nlsk_fd);
 472         return ret;
 473 }
 474
 475 /* Try to disable ROCE by sysfs. */
 476 static int
 477 mlx5_vdpa_sys_roce_disable(const char *addr)
 478 {
 479         FILE *file_o;
 480         int enable;
 481         int ret;
 482
 483         MKSTR(file_p, "/sys/bus/pci/devices/%s/roce_enable", addr);
 484         file_o = fopen(file_p, "rb");
 485         if (!file_o) {
 486                 rte_errno = ENOTSUP;
 487                 return -ENOTSUP;
 488         }
 489         ret = fscanf(file_o, "%d", &enable);
 490         if (ret != 1) {
 491                 rte_errno = EINVAL;
 492                 ret = EINVAL;
 493                 goto close;
 494         } else if (!enable) {
 495                 ret = 0;
 496                 DRV_LOG(INFO, "ROCE has already disabled(sysfs).");
 497                 goto close;
 498         }
 499         fclose(file_o);
 500         file_o = fopen(file_p, "wb");
 501         if (!file_o) {
 502                 rte_errno = ENOTSUP;
 503                 return -ENOTSUP;
 504         }
 505         fprintf(file_o, "0\n");
 506         ret = 0;
 507 close:
 508         if (ret)
 509                 DRV_LOG(DEBUG, "Failed to disable ROCE by sysfs: %d.", ret);
 510         else
 511                 DRV_LOG(INFO, "ROCE is disabled by sysfs successfully.");
 512         fclose(file_o);
 513         return ret;
 514 }
 515
 516 #define MLX5_VDPA_MAX_RETRIES 20
 517 #define MLX5_VDPA_USEC 1000
 518 static int
 519 mlx5_vdpa_roce_disable(struct rte_pci_addr *addr, struct ibv_device **ibv)
 520 {
 521         char addr_name[64] = {0};
 522
 523         rte_pci_device_name(addr, addr_name, sizeof(addr_name));
 524         /* Firstly try to disable ROCE by Netlink and fallback to sysfs. */
 525         if (mlx5_vdpa_nl_roce_disable(addr_name) == 0 ||
 526             mlx5_vdpa_sys_roce_disable(addr_name) == 0) {
 527                 /*
 528                  * Succeed to disable ROCE, wait for the IB device to appear
 529                  * again after reload.
 530                  */
 531                 int r;
 532                 struct ibv_device *ibv_new;
 533
 534                 for (r = MLX5_VDPA_MAX_RETRIES; r; r--) {
 535                         ibv_new = mlx5_vdpa_get_ib_device_match(addr);
 536                         if (ibv_new) {
 537                                 *ibv = ibv_new;
 538                                 return 0;
 539                         }
 540                         usleep(MLX5_VDPA_USEC);
 541                 }
 542                 DRV_LOG(ERR, "Cannot much device %s after ROCE disable, "
 543                         "retries exceed %d", addr_name, MLX5_VDPA_MAX_RETRIES);
 544                 rte_errno = EAGAIN;
 545         }
 546         return -rte_errno;
 547 }
 548
 549 /**
 550  * DPDK callback to register a PCI device.
 551  *
 552  * This function spawns vdpa device out of a given PCI device.
 553  *
 554  * @param[in] pci_drv
 555  *   PCI driver structure (mlx5_vpda_driver).
 556  * @param[in] pci_dev
 557  *   PCI device information.
 558  *
 559  * @return
 560  *   0 on success, 1 to skip this driver, a negative errno value otherwise
 561  *   and rte_errno is set.
 562  */
 563 static int
 564 mlx5_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 565                     struct rte_pci_device *pci_dev __rte_unused)
 566 {
 567         struct ibv_device *ibv;
 568         struct mlx5_vdpa_priv *priv = NULL;
 569         struct ibv_context *ctx = NULL;
 570         struct mlx5_hca_attr attr;
 571         int ret;
 572
 573         if (mlx5_class_get(pci_dev->device.devargs) != MLX5_CLASS_VDPA) {
 574                 DRV_LOG(DEBUG, "Skip probing - should be probed by other mlx5"
 575                         " driver.");
 576                 return 1;
 577         }
 578         ibv = mlx5_vdpa_get_ib_device_match(&pci_dev->addr);
 579         if (!ibv) {
 580                 DRV_LOG(ERR, "No matching IB device for PCI slot "
 581                         PCI_PRI_FMT ".", pci_dev->addr.domain,
 582                         pci_dev->addr.bus, pci_dev->addr.devid,
 583                         pci_dev->addr.function);
 584                 return -rte_errno;
 585         } else {
 586                 DRV_LOG(INFO, "PCI information matches for device \"%s\".",
 587                         ibv->name);
 588         }
 589         if (mlx5_vdpa_roce_disable(&pci_dev->addr, &ibv) != 0) {
 590                 DRV_LOG(WARNING, "Failed to disable ROCE for \"%s\".",
 591                         ibv->name);
 592                 return -rte_errno;
 593         }
 594         ctx = mlx5_glue->dv_open_device(ibv);
 595         if (!ctx) {
 596                 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
 597                 rte_errno = ENODEV;
 598                 return -rte_errno;
 599         }
 600         ret = mlx5_devx_cmd_query_hca_attr(ctx, &attr);
 601         if (ret) {
 602                 DRV_LOG(ERR, "Unable to read HCA capabilities.");
 603                 rte_errno = ENOTSUP;
 604                 goto error;
 605         } else if (!attr.vdpa.valid || !attr.vdpa.max_num_virtio_queues) {
 606                 DRV_LOG(ERR, "Not enough capabilities to support vdpa, maybe "
 607                         "old FW/OFED version?");
 608                 rte_errno = ENOTSUP;
 609                 goto error;
 610         }
 611         if (!attr.vdpa.queue_counters_valid)
 612                 DRV_LOG(DEBUG, "No capability to support virtq statistics.");
 613         priv = rte_zmalloc("mlx5 vDPA device private", sizeof(*priv) +
 614                            sizeof(struct mlx5_vdpa_virtq) *
 615                            attr.vdpa.max_num_virtio_queues * 2,
 616                            RTE_CACHE_LINE_SIZE);
 617         if (!priv) {
 618                 DRV_LOG(ERR, "Failed to allocate private memory.");
 619                 rte_errno = ENOMEM;
 620                 goto error;
 621         }
 622         priv->caps = attr.vdpa;
 623         priv->log_max_rqt_size = attr.log_max_rqt_size;
 624         priv->ctx = ctx;
 625         priv->dev_addr.pci_addr = pci_dev->addr;
 626         priv->dev_addr.type = VDPA_ADDR_PCI;
 627         priv->var = mlx5_glue->dv_alloc_var(ctx, 0);
 628         if (!priv->var) {
 629                 DRV_LOG(ERR, "Failed to allocate VAR %u.\n", errno);
 630                 goto error;
 631         }
 632         priv->id = rte_vdpa_register_device(&priv->dev_addr, &mlx5_vdpa_ops);
 633         if (priv->id < 0) {
 634                 DRV_LOG(ERR, "Failed to register vDPA device.");
 635                 rte_errno = rte_errno ? rte_errno : EINVAL;
 636                 goto error;
 637         }
 638         SLIST_INIT(&priv->mr_list);
 639         pthread_mutex_lock(&priv_list_lock);
 640         TAILQ_INSERT_TAIL(&priv_list, priv, next);
 641         pthread_mutex_unlock(&priv_list_lock);
 642         return 0;
 643
 644 error:
 645         if (priv) {
 646                 if (priv->var)
 647                         mlx5_glue->dv_free_var(priv->var);
 648                 rte_free(priv);
 649         }
 650         if (ctx)
 651                 mlx5_glue->close_device(ctx);
 652         return -rte_errno;
 653 }
 654
 655 /**
 656  * DPDK callback to remove a PCI device.
 657  *
 658  * This function removes all vDPA devices belong to a given PCI device.
 659  *
 660  * @param[in] pci_dev
 661  *   Pointer to the PCI device.
 662  *
 663  * @return
 664  *   0 on success, the function cannot fail.
 665  */
 666 static int
 667 mlx5_vdpa_pci_remove(struct rte_pci_device *pci_dev)
 668 {
 669         struct mlx5_vdpa_priv *priv = NULL;
 670         int found = 0;
 671
 672         pthread_mutex_lock(&priv_list_lock);
 673         TAILQ_FOREACH(priv, &priv_list, next) {
 674                 if (memcmp(&priv->dev_addr.pci_addr, &pci_dev->addr,
 675                            sizeof(pci_dev->addr)) == 0) {
 676                         found = 1;
 677                         break;
 678                 }
 679         }
 680         if (found)
 681                 TAILQ_REMOVE(&priv_list, priv, next);
 682         pthread_mutex_unlock(&priv_list_lock);
 683         if (found) {
 684                 if (priv->configured)
 685                         mlx5_vdpa_dev_close(priv->vid);
 686                 if (priv->var) {
 687                         mlx5_glue->dv_free_var(priv->var);
 688                         priv->var = NULL;
 689                 }
 690                 mlx5_glue->close_device(priv->ctx);
 691                 rte_free(priv);
 692         }
 693         return 0;
 694 }
 695
 696 static const struct rte_pci_id mlx5_vdpa_pci_id_map[] = {
 697         {
 698                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 699                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 700         },
 701         {
 702                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 703                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF)
 704         },
 705         {
 706                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 707                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
 708         },
 709         {
 710                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 711                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF)
 712         },
 713         {
 714                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 715                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
 716         },
 717         {
 718                 .vendor_id = 0
 719         }
 720 };
 721
 722 static struct rte_pci_driver mlx5_vdpa_driver = {
 723         .driver = {
 724                 .name = "mlx5_vdpa",
 725         },
 726         .id_table = mlx5_vdpa_pci_id_map,
 727         .probe = mlx5_vdpa_pci_probe,
 728         .remove = mlx5_vdpa_pci_remove,
 729         .drv_flags = 0,
 730 };
 731
 732 /**
 733  * Driver initialization routine.
 734  */
 735 RTE_INIT(rte_mlx5_vdpa_init)
 736 {
 737         /* Initialize common log type. */
 738         mlx5_vdpa_logtype = rte_log_register("pmd.vdpa.mlx5");
 739         if (mlx5_vdpa_logtype >= 0)
 740                 rte_log_set_level(mlx5_vdpa_logtype, RTE_LOG_NOTICE);
 741         if (mlx5_glue)
 742                 rte_pci_register(&mlx5_vdpa_driver);
 743 }
 744
 745 RTE_PMD_EXPORT_NAME(net_mlx5_vdpa, __COUNTER__);
 746 RTE_PMD_REGISTER_PCI_TABLE(net_mlx5_vdpa, mlx5_vdpa_pci_id_map);
 747 RTE_PMD_REGISTER_KMOD_DEP(net_mlx5_vdpa, "* ib_uverbs & mlx5_core & mlx5_ib");