drivers/vdpa/mlx5/mlx5_vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <unistd.h>
   5 #include <net/if.h>
   6 #include <sys/socket.h>
   7 #include <sys/ioctl.h>
   8 #include <fcntl.h>
   9 #include <netinet/in.h>
  10
  11 #include <rte_malloc.h>
  12 #include <rte_log.h>
  13 #include <rte_errno.h>
  14 #include <rte_bus_pci.h>
  15 #include <rte_pci.h>
  16 #include <rte_string_fns.h>
  17
  18 #include <mlx5_glue.h>
  19 #include <mlx5_common.h>
  20 #include <mlx5_devx_cmds.h>
  21 #include <mlx5_prm.h>
  22 #include <mlx5_nl.h>
  23
  24 #include "mlx5_vdpa_utils.h"
  25 #include "mlx5_vdpa.h"
  26
  27
  28 #define MLX5_VDPA_DEFAULT_FEATURES ((1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
  29                             (1ULL << VIRTIO_F_ANY_LAYOUT) | \
  30                             (1ULL << VIRTIO_NET_F_MQ) | \
  31                             (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \
  32                             (1ULL << VIRTIO_F_ORDER_PLATFORM) | \
  33                             (1ULL << VHOST_F_LOG_ALL) | \
  34                             (1ULL << VIRTIO_NET_F_MTU))
  35
  36 #define MLX5_VDPA_PROTOCOL_FEATURES \
  37                             ((1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
  38                              (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
  39                              (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
  40                              (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) | \
  41                              (1ULL << VHOST_USER_PROTOCOL_F_MQ) | \
  42                              (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))
  43
  44 #define MLX5_VDPA_MAX_RETRIES 20
  45 #define MLX5_VDPA_USEC 1000
  46 #define MLX5_VDPA_DEFAULT_NO_TRAFFIC_TIME_S 2LLU
  47
  48 TAILQ_HEAD(mlx5_vdpa_privs, mlx5_vdpa_priv) priv_list =
  49                                               TAILQ_HEAD_INITIALIZER(priv_list);
  50 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
  51 int mlx5_vdpa_logtype;
  52
  53 static struct mlx5_vdpa_priv *
  54 mlx5_vdpa_find_priv_resource_by_vdev(struct rte_vdpa_device *vdev)
  55 {
  56         struct mlx5_vdpa_priv *priv;
  57         int found = 0;
  58
  59         pthread_mutex_lock(&priv_list_lock);
  60         TAILQ_FOREACH(priv, &priv_list, next) {
  61                 if (vdev == priv->vdev) {
  62                         found = 1;
  63                         break;
  64                 }
  65         }
  66         pthread_mutex_unlock(&priv_list_lock);
  67         if (!found) {
  68                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
  69                 rte_errno = EINVAL;
  70                 return NULL;
  71         }
  72         return priv;
  73 }
  74
  75 static int
  76 mlx5_vdpa_get_queue_num(struct rte_vdpa_device *vdev, uint32_t *queue_num)
  77 {
  78         struct mlx5_vdpa_priv *priv =
  79                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
  80
  81         if (priv == NULL) {
  82                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
  83                 return -1;
  84         }
  85         *queue_num = priv->caps.max_num_virtio_queues;
  86         return 0;
  87 }
  88
  89 static int
  90 mlx5_vdpa_get_vdpa_features(struct rte_vdpa_device *vdev, uint64_t *features)
  91 {
  92         struct mlx5_vdpa_priv *priv =
  93                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
  94
  95         if (priv == NULL) {
  96                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
  97                 return -1;
  98         }
  99         *features = MLX5_VDPA_DEFAULT_FEATURES;
 100         if (priv->caps.virtio_queue_type & (1 << MLX5_VIRTQ_TYPE_PACKED))
 101                 *features |= (1ULL << VIRTIO_F_RING_PACKED);
 102         if (priv->caps.tso_ipv4)
 103                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO4);
 104         if (priv->caps.tso_ipv6)
 105                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO6);
 106         if (priv->caps.tx_csum)
 107                 *features |= (1ULL << VIRTIO_NET_F_CSUM);
 108         if (priv->caps.rx_csum)
 109                 *features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
 110         if (priv->caps.virtio_version_1_0)
 111                 *features |= (1ULL << VIRTIO_F_VERSION_1);
 112         return 0;
 113 }
 114
 115 static int
 116 mlx5_vdpa_get_protocol_features(struct rte_vdpa_device *vdev,
 117                 uint64_t *features)
 118 {
 119         struct mlx5_vdpa_priv *priv =
 120                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 121
 122         if (priv == NULL) {
 123                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 124                 return -1;
 125         }
 126         *features = MLX5_VDPA_PROTOCOL_FEATURES;
 127         return 0;
 128 }
 129
 130 static int
 131 mlx5_vdpa_set_vring_state(int vid, int vring, int state)
 132 {
 133         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 134         struct mlx5_vdpa_priv *priv =
 135                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 136
 137         if (priv == NULL) {
 138                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 139                 return -EINVAL;
 140         }
 141         if (vring >= (int)priv->caps.max_num_virtio_queues * 2) {
 142                 DRV_LOG(ERR, "Too big vring id: %d.", vring);
 143                 return -E2BIG;
 144         }
 145         return mlx5_vdpa_virtq_enable(priv, vring, state);
 146 }
 147
 148 static int
 149 mlx5_vdpa_direct_db_prepare(struct mlx5_vdpa_priv *priv)
 150 {
 151         int ret;
 152
 153         if (priv->direct_notifier) {
 154                 ret = rte_vhost_host_notifier_ctrl(priv->vid,
 155                                                    RTE_VHOST_QUEUE_ALL, false);
 156                 if (ret != 0) {
 157                         DRV_LOG(INFO, "Direct HW notifier FD cannot be "
 158                                 "destroyed for device %d: %d.", priv->vid, ret);
 159                         return -1;
 160                 }
 161                 priv->direct_notifier = 0;
 162         }
 163         ret = rte_vhost_host_notifier_ctrl(priv->vid, RTE_VHOST_QUEUE_ALL,
 164                                            true);
 165         if (ret != 0)
 166                 DRV_LOG(INFO, "Direct HW notifier FD cannot be configured for"
 167                         " device %d: %d.", priv->vid, ret);
 168         else
 169                 priv->direct_notifier = 1;
 170         return 0;
 171 }
 172
 173 static int
 174 mlx5_vdpa_features_set(int vid)
 175 {
 176         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 177         struct mlx5_vdpa_priv *priv =
 178                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 179         uint64_t log_base, log_size;
 180         uint64_t features;
 181         int ret;
 182
 183         if (priv == NULL) {
 184                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 185                 return -EINVAL;
 186         }
 187         ret = rte_vhost_get_negotiated_features(vid, &features);
 188         if (ret) {
 189                 DRV_LOG(ERR, "Failed to get negotiated features.");
 190                 return ret;
 191         }
 192         if (RTE_VHOST_NEED_LOG(features)) {
 193                 ret = rte_vhost_get_log_base(vid, &log_base, &log_size);
 194                 if (ret) {
 195                         DRV_LOG(ERR, "Failed to get log base.");
 196                         return ret;
 197                 }
 198                 ret = mlx5_vdpa_dirty_bitmap_set(priv, log_base, log_size);
 199                 if (ret) {
 200                         DRV_LOG(ERR, "Failed to set dirty bitmap.");
 201                         return ret;
 202                 }
 203                 DRV_LOG(INFO, "mlx5 vdpa: enabling dirty logging...");
 204                 ret = mlx5_vdpa_logging_enable(priv, 1);
 205                 if (ret) {
 206                         DRV_LOG(ERR, "Failed t enable dirty logging.");
 207                         return ret;
 208                 }
 209         }
 210         return 0;
 211 }
 212
 213 static int
 214 mlx5_vdpa_pd_create(struct mlx5_vdpa_priv *priv)
 215 {
 216 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 217         priv->pd = mlx5_glue->alloc_pd(priv->ctx);
 218         if (priv->pd == NULL) {
 219                 DRV_LOG(ERR, "Failed to allocate PD.");
 220                 return errno ? -errno : -ENOMEM;
 221         }
 222         struct mlx5dv_obj obj;
 223         struct mlx5dv_pd pd_info;
 224         int ret = 0;
 225
 226         obj.pd.in = priv->pd;
 227         obj.pd.out = &pd_info;
 228         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
 229         if (ret) {
 230                 DRV_LOG(ERR, "Fail to get PD object info.");
 231                 mlx5_glue->dealloc_pd(priv->pd);
 232                 priv->pd = NULL;
 233                 return -errno;
 234         }
 235         priv->pdn = pd_info.pdn;
 236         return 0;
 237 #else
 238         (void)priv;
 239         DRV_LOG(ERR, "Cannot get pdn - no DV support.");
 240         return -ENOTSUP;
 241 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
 242 }
 243
 244 static int
 245 mlx5_vdpa_mtu_set(struct mlx5_vdpa_priv *priv)
 246 {
 247         struct ifreq request;
 248         uint16_t vhost_mtu = 0;
 249         uint16_t kern_mtu = 0;
 250         int ret = rte_vhost_get_mtu(priv->vid, &vhost_mtu);
 251         int sock;
 252         int retries = MLX5_VDPA_MAX_RETRIES;
 253
 254         if (ret) {
 255                 DRV_LOG(DEBUG, "Cannot get vhost MTU - %d.", ret);
 256                 return ret;
 257         }
 258         if (!vhost_mtu) {
 259                 DRV_LOG(DEBUG, "Vhost MTU is 0.");
 260                 return ret;
 261         }
 262         ret = mlx5_get_ifname_sysfs(priv->ctx->device->ibdev_path,
 263                                     request.ifr_name);
 264         if (ret) {
 265                 DRV_LOG(DEBUG, "Cannot get kernel IF name - %d.", ret);
 266                 return ret;
 267         }
 268         sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
 269         if (sock == -1) {
 270                 DRV_LOG(DEBUG, "Cannot open IF socket.");
 271                 return sock;
 272         }
 273         while (retries--) {
 274                 ret = ioctl(sock, SIOCGIFMTU, &request);
 275                 if (ret == -1)
 276                         break;
 277                 kern_mtu = request.ifr_mtu;
 278                 DRV_LOG(DEBUG, "MTU: current %d requested %d.", (int)kern_mtu,
 279                         (int)vhost_mtu);
 280                 if (kern_mtu == vhost_mtu)
 281                         break;
 282                 request.ifr_mtu = vhost_mtu;
 283                 ret = ioctl(sock, SIOCSIFMTU, &request);
 284                 if (ret == -1)
 285                         break;
 286                 request.ifr_mtu = 0;
 287                 usleep(MLX5_VDPA_USEC);
 288         }
 289         close(sock);
 290         return kern_mtu == vhost_mtu ? 0 : -1;
 291 }
 292
 293 static int
 294 mlx5_vdpa_dev_close(int vid)
 295 {
 296         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 297         struct mlx5_vdpa_priv *priv =
 298                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 299         int ret = 0;
 300
 301         if (priv == NULL) {
 302                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 303                 return -1;
 304         }
 305         if (priv->configured)
 306                 ret |= mlx5_vdpa_lm_log(priv);
 307         mlx5_vdpa_cqe_event_unset(priv);
 308         mlx5_vdpa_steer_unset(priv);
 309         mlx5_vdpa_virtqs_release(priv);
 310         mlx5_vdpa_event_qp_global_release(priv);
 311         mlx5_vdpa_mem_dereg(priv);
 312         if (priv->pd) {
 313                 claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 314                 priv->pd = NULL;
 315         }
 316         priv->configured = 0;
 317         priv->vid = 0;
 318         DRV_LOG(INFO, "vDPA device %d was closed.", vid);
 319         return ret;
 320 }
 321
 322 static int
 323 mlx5_vdpa_dev_config(int vid)
 324 {
 325         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 326         struct mlx5_vdpa_priv *priv =
 327                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 328
 329         if (priv == NULL) {
 330                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 331                 return -EINVAL;
 332         }
 333         if (priv->configured && mlx5_vdpa_dev_close(vid)) {
 334                 DRV_LOG(ERR, "Failed to reconfigure vid %d.", vid);
 335                 return -1;
 336         }
 337         priv->vid = vid;
 338         if (mlx5_vdpa_mtu_set(priv))
 339                 DRV_LOG(WARNING, "MTU cannot be set on device %s.",
 340                                 vdev->device->name);
 341         if (mlx5_vdpa_pd_create(priv) || mlx5_vdpa_mem_register(priv) ||
 342             mlx5_vdpa_direct_db_prepare(priv) ||
 343             mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) ||
 344             mlx5_vdpa_cqe_event_setup(priv)) {
 345                 mlx5_vdpa_dev_close(vid);
 346                 return -1;
 347         }
 348         priv->configured = 1;
 349         DRV_LOG(INFO, "vDPA device %d was configured.", vid);
 350         return 0;
 351 }
 352
 353 static int
 354 mlx5_vdpa_get_device_fd(int vid)
 355 {
 356         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 357         struct mlx5_vdpa_priv *priv =
 358                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 359
 360         if (priv == NULL) {
 361                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 362                 return -EINVAL;
 363         }
 364         return priv->ctx->cmd_fd;
 365 }
 366
 367 static int
 368 mlx5_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
 369 {
 370         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 371         struct mlx5_vdpa_priv *priv =
 372                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 373
 374         RTE_SET_USED(qid);
 375         if (priv == NULL) {
 376                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 377                 return -EINVAL;
 378         }
 379         if (!priv->var) {
 380                 DRV_LOG(ERR, "VAR was not created for device %s, is the device"
 381                         " configured?.", vdev->device->name);
 382                 return -EINVAL;
 383         }
 384         *offset = priv->var->mmap_off;
 385         *size = priv->var->length;
 386         return 0;
 387 }
 388
 389 static int
 390 mlx5_vdpa_get_stats_names(struct rte_vdpa_device *vdev,
 391                 struct rte_vdpa_stat_name *stats_names,
 392                 unsigned int size)
 393 {
 394         static const char *mlx5_vdpa_stats_names[MLX5_VDPA_STATS_MAX] = {
 395                 "received_descriptors",
 396                 "completed_descriptors",
 397                 "bad descriptor errors",
 398                 "exceed max chain",
 399                 "invalid buffer",
 400                 "completion errors",
 401         };
 402         struct mlx5_vdpa_priv *priv =
 403                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 404         unsigned int i;
 405
 406         if (priv == NULL) {
 407                 DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
 408                 return -ENODEV;
 409         }
 410         if (!stats_names)
 411                 return MLX5_VDPA_STATS_MAX;
 412         size = RTE_MIN(size, (unsigned int)MLX5_VDPA_STATS_MAX);
 413         for (i = 0; i < size; ++i)
 414                 strlcpy(stats_names[i].name, mlx5_vdpa_stats_names[i],
 415                         RTE_VDPA_STATS_NAME_SIZE);
 416         return size;
 417 }
 418
 419 static int
 420 mlx5_vdpa_get_stats(struct rte_vdpa_device *vdev, int qid,
 421                 struct rte_vdpa_stat *stats, unsigned int n)
 422 {
 423         struct mlx5_vdpa_priv *priv =
 424                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 425
 426         if (priv == NULL) {
 427                 DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
 428                 return -ENODEV;
 429         }
 430         if (!priv->configured) {
 431                 DRV_LOG(ERR, "Device %s was not configured.",
 432                                 vdev->device->name);
 433                 return -ENODATA;
 434         }
 435         if (qid >= (int)priv->nr_virtqs) {
 436                 DRV_LOG(ERR, "Too big vring id: %d for device %s.", qid,
 437                                 vdev->device->name);
 438                 return -E2BIG;
 439         }
 440         if (!priv->caps.queue_counters_valid) {
 441                 DRV_LOG(ERR, "Virtq statistics is not supported for device %s.",
 442                         vdev->device->name);
 443                 return -ENOTSUP;
 444         }
 445         return mlx5_vdpa_virtq_stats_get(priv, qid, stats, n);
 446 }
 447
 448 static int
 449 mlx5_vdpa_reset_stats(struct rte_vdpa_device *vdev, int qid)
 450 {
 451         struct mlx5_vdpa_priv *priv =
 452                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 453
 454         if (priv == NULL) {
 455                 DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
 456                 return -ENODEV;
 457         }
 458         if (!priv->configured) {
 459                 DRV_LOG(ERR, "Device %s was not configured.",
 460                                 vdev->device->name);
 461                 return -ENODATA;
 462         }
 463         if (qid >= (int)priv->nr_virtqs) {
 464                 DRV_LOG(ERR, "Too big vring id: %d for device %s.", qid,
 465                                 vdev->device->name);
 466                 return -E2BIG;
 467         }
 468         if (!priv->caps.queue_counters_valid) {
 469                 DRV_LOG(ERR, "Virtq statistics is not supported for device %s.",
 470                         vdev->device->name);
 471                 return -ENOTSUP;
 472         }
 473         return mlx5_vdpa_virtq_stats_reset(priv, qid);
 474 }
 475
 476 static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {
 477         .get_queue_num = mlx5_vdpa_get_queue_num,
 478         .get_features = mlx5_vdpa_get_vdpa_features,
 479         .get_protocol_features = mlx5_vdpa_get_protocol_features,
 480         .dev_conf = mlx5_vdpa_dev_config,
 481         .dev_close = mlx5_vdpa_dev_close,
 482         .set_vring_state = mlx5_vdpa_set_vring_state,
 483         .set_features = mlx5_vdpa_features_set,
 484         .migration_done = NULL,
 485         .get_vfio_group_fd = NULL,
 486         .get_vfio_device_fd = mlx5_vdpa_get_device_fd,
 487         .get_notify_area = mlx5_vdpa_get_notify_area,
 488         .get_stats_names = mlx5_vdpa_get_stats_names,
 489         .get_stats = mlx5_vdpa_get_stats,
 490         .reset_stats = mlx5_vdpa_reset_stats,
 491 };
 492
 493 static struct ibv_device *
 494 mlx5_vdpa_get_ib_device_match(struct rte_pci_addr *addr)
 495 {
 496         int n;
 497         struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
 498         struct ibv_device *ibv_match = NULL;
 499
 500         if (!ibv_list) {
 501                 rte_errno = ENOSYS;
 502                 return NULL;
 503         }
 504         while (n-- > 0) {
 505                 struct rte_pci_addr pci_addr;
 506
 507                 DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
 508                 if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &pci_addr))
 509                         continue;
 510                 if (rte_pci_addr_cmp(addr, &pci_addr))
 511                         continue;
 512                 ibv_match = ibv_list[n];
 513                 break;
 514         }
 515         if (!ibv_match)
 516                 rte_errno = ENOENT;
 517         mlx5_glue->free_device_list(ibv_list);
 518         return ibv_match;
 519 }
 520
 521 /* Try to disable ROCE by Netlink\Devlink. */
 522 static int
 523 mlx5_vdpa_nl_roce_disable(const char *addr)
 524 {
 525         int nlsk_fd = mlx5_nl_init(NETLINK_GENERIC);
 526         int devlink_id;
 527         int enable;
 528         int ret;
 529
 530         if (nlsk_fd < 0)
 531                 return nlsk_fd;
 532         devlink_id = mlx5_nl_devlink_family_id_get(nlsk_fd);
 533         if (devlink_id < 0) {
 534                 ret = devlink_id;
 535                 DRV_LOG(DEBUG, "Failed to get devlink id for ROCE operations by"
 536                         " Netlink.");
 537                 goto close;
 538         }
 539         ret = mlx5_nl_enable_roce_get(nlsk_fd, devlink_id, addr, &enable);
 540         if (ret) {
 541                 DRV_LOG(DEBUG, "Failed to get ROCE enable by Netlink: %d.",
 542                         ret);
 543                 goto close;
 544         } else if (!enable) {
 545                 DRV_LOG(INFO, "ROCE has already disabled(Netlink).");
 546                 goto close;
 547         }
 548         ret = mlx5_nl_enable_roce_set(nlsk_fd, devlink_id, addr, 0);
 549         if (ret)
 550                 DRV_LOG(DEBUG, "Failed to disable ROCE by Netlink: %d.", ret);
 551         else
 552                 DRV_LOG(INFO, "ROCE is disabled by Netlink successfully.");
 553 close:
 554         close(nlsk_fd);
 555         return ret;
 556 }
 557
 558 /* Try to disable ROCE by sysfs. */
 559 static int
 560 mlx5_vdpa_sys_roce_disable(const char *addr)
 561 {
 562         FILE *file_o;
 563         int enable;
 564         int ret;
 565
 566         MKSTR(file_p, "/sys/bus/pci/devices/%s/roce_enable", addr);
 567         file_o = fopen(file_p, "rb");
 568         if (!file_o) {
 569                 rte_errno = ENOTSUP;
 570                 return -ENOTSUP;
 571         }
 572         ret = fscanf(file_o, "%d", &enable);
 573         if (ret != 1) {
 574                 rte_errno = EINVAL;
 575                 ret = EINVAL;
 576                 goto close;
 577         } else if (!enable) {
 578                 ret = 0;
 579                 DRV_LOG(INFO, "ROCE has already disabled(sysfs).");
 580                 goto close;
 581         }
 582         fclose(file_o);
 583         file_o = fopen(file_p, "wb");
 584         if (!file_o) {
 585                 rte_errno = ENOTSUP;
 586                 return -ENOTSUP;
 587         }
 588         fprintf(file_o, "0\n");
 589         ret = 0;
 590 close:
 591         if (ret)
 592                 DRV_LOG(DEBUG, "Failed to disable ROCE by sysfs: %d.", ret);
 593         else
 594                 DRV_LOG(INFO, "ROCE is disabled by sysfs successfully.");
 595         fclose(file_o);
 596         return ret;
 597 }
 598
 599 static int
 600 mlx5_vdpa_roce_disable(struct rte_pci_addr *addr, struct ibv_device **ibv)
 601 {
 602         char addr_name[64] = {0};
 603
 604         rte_pci_device_name(addr, addr_name, sizeof(addr_name));
 605         /* Firstly try to disable ROCE by Netlink and fallback to sysfs. */
 606         if (mlx5_vdpa_nl_roce_disable(addr_name) == 0 ||
 607             mlx5_vdpa_sys_roce_disable(addr_name) == 0) {
 608                 /*
 609                  * Succeed to disable ROCE, wait for the IB device to appear
 610                  * again after reload.
 611                  */
 612                 int r;
 613                 struct ibv_device *ibv_new;
 614
 615                 for (r = MLX5_VDPA_MAX_RETRIES; r; r--) {
 616                         ibv_new = mlx5_vdpa_get_ib_device_match(addr);
 617                         if (ibv_new) {
 618                                 *ibv = ibv_new;
 619                                 return 0;
 620                         }
 621                         usleep(MLX5_VDPA_USEC);
 622                 }
 623                 DRV_LOG(ERR, "Cannot much device %s after ROCE disable, "
 624                         "retries exceed %d", addr_name, MLX5_VDPA_MAX_RETRIES);
 625                 rte_errno = EAGAIN;
 626         }
 627         return -rte_errno;
 628 }
 629
 630 static int
 631 mlx5_vdpa_args_check_handler(const char *key, const char *val, void *opaque)
 632 {
 633         struct mlx5_vdpa_priv *priv = opaque;
 634         unsigned long tmp;
 635
 636         if (strcmp(key, "class") == 0)
 637                 return 0;
 638         errno = 0;
 639         tmp = strtoul(val, NULL, 0);
 640         if (errno) {
 641                 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 642                 return -errno;
 643         }
 644         if (strcmp(key, "event_mode") == 0) {
 645                 if (tmp <= MLX5_VDPA_EVENT_MODE_ONLY_INTERRUPT)
 646                         priv->event_mode = (int)tmp;
 647                 else
 648                         DRV_LOG(WARNING, "Invalid event_mode %s.", val);
 649         } else if (strcmp(key, "event_us") == 0) {
 650                 priv->event_us = (uint32_t)tmp;
 651         } else if (strcmp(key, "no_traffic_time") == 0) {
 652                 priv->no_traffic_time_s = (uint32_t)tmp;
 653         } else {
 654                 DRV_LOG(WARNING, "Invalid key %s.", key);
 655         }
 656         return 0;
 657 }
 658
 659 static void
 660 mlx5_vdpa_config_get(struct rte_devargs *devargs, struct mlx5_vdpa_priv *priv)
 661 {
 662         struct rte_kvargs *kvlist;
 663
 664         priv->event_mode = MLX5_VDPA_EVENT_MODE_DYNAMIC_TIMER;
 665         priv->event_us = 0;
 666         priv->no_traffic_time_s = MLX5_VDPA_DEFAULT_NO_TRAFFIC_TIME_S;
 667         if (devargs == NULL)
 668                 return;
 669         kvlist = rte_kvargs_parse(devargs->args, NULL);
 670         if (kvlist == NULL)
 671                 return;
 672         rte_kvargs_process(kvlist, NULL, mlx5_vdpa_args_check_handler, priv);
 673         rte_kvargs_free(kvlist);
 674         if (!priv->event_us) {
 675                 if (priv->event_mode == MLX5_VDPA_EVENT_MODE_DYNAMIC_TIMER)
 676                         priv->event_us = MLX5_VDPA_DEFAULT_TIMER_STEP_US;
 677                 else if (priv->event_mode == MLX5_VDPA_EVENT_MODE_FIXED_TIMER)
 678                         priv->event_us = MLX5_VDPA_DEFAULT_TIMER_DELAY_US;
 679         }
 680         DRV_LOG(DEBUG, "event mode is %d.", priv->event_mode);
 681         DRV_LOG(DEBUG, "event_us is %u us.", priv->event_us);
 682         DRV_LOG(DEBUG, "no traffic time is %u s.", priv->no_traffic_time_s);
 683 }
 684
 685 /**
 686  * DPDK callback to register a PCI device.
 687  *
 688  * This function spawns vdpa device out of a given PCI device.
 689  *
 690  * @param[in] pci_drv
 691  *   PCI driver structure (mlx5_vpda_driver).
 692  * @param[in] pci_dev
 693  *   PCI device information.
 694  *
 695  * @return
 696  *   0 on success, 1 to skip this driver, a negative errno value otherwise
 697  *   and rte_errno is set.
 698  */
 699 static int
 700 mlx5_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 701                     struct rte_pci_device *pci_dev __rte_unused)
 702 {
 703         struct ibv_device *ibv;
 704         struct mlx5_vdpa_priv *priv = NULL;
 705         struct ibv_context *ctx = NULL;
 706         struct mlx5_hca_attr attr;
 707         int ret;
 708
 709         if (mlx5_class_get(pci_dev->device.devargs) != MLX5_CLASS_VDPA) {
 710                 DRV_LOG(DEBUG, "Skip probing - should be probed by other mlx5"
 711                         " driver.");
 712                 return 1;
 713         }
 714         ibv = mlx5_vdpa_get_ib_device_match(&pci_dev->addr);
 715         if (!ibv) {
 716                 DRV_LOG(ERR, "No matching IB device for PCI slot "
 717                         PCI_PRI_FMT ".", pci_dev->addr.domain,
 718                         pci_dev->addr.bus, pci_dev->addr.devid,
 719                         pci_dev->addr.function);
 720                 return -rte_errno;
 721         } else {
 722                 DRV_LOG(INFO, "PCI information matches for device \"%s\".",
 723                         ibv->name);
 724         }
 725         if (mlx5_vdpa_roce_disable(&pci_dev->addr, &ibv) != 0) {
 726                 DRV_LOG(WARNING, "Failed to disable ROCE for \"%s\".",
 727                         ibv->name);
 728                 return -rte_errno;
 729         }
 730         ctx = mlx5_glue->dv_open_device(ibv);
 731         if (!ctx) {
 732                 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
 733                 rte_errno = ENODEV;
 734                 return -rte_errno;
 735         }
 736         ret = mlx5_devx_cmd_query_hca_attr(ctx, &attr);
 737         if (ret) {
 738                 DRV_LOG(ERR, "Unable to read HCA capabilities.");
 739                 rte_errno = ENOTSUP;
 740                 goto error;
 741         } else if (!attr.vdpa.valid || !attr.vdpa.max_num_virtio_queues) {
 742                 DRV_LOG(ERR, "Not enough capabilities to support vdpa, maybe "
 743                         "old FW/OFED version?");
 744                 rte_errno = ENOTSUP;
 745                 goto error;
 746         }
 747         if (!attr.vdpa.queue_counters_valid)
 748                 DRV_LOG(DEBUG, "No capability to support virtq statistics.");
 749         priv = rte_zmalloc("mlx5 vDPA device private", sizeof(*priv) +
 750                            sizeof(struct mlx5_vdpa_virtq) *
 751                            attr.vdpa.max_num_virtio_queues * 2,
 752                            RTE_CACHE_LINE_SIZE);
 753         if (!priv) {
 754                 DRV_LOG(ERR, "Failed to allocate private memory.");
 755                 rte_errno = ENOMEM;
 756                 goto error;
 757         }
 758         priv->caps = attr.vdpa;
 759         priv->log_max_rqt_size = attr.log_max_rqt_size;
 760         priv->ctx = ctx;
 761         priv->pci_dev = pci_dev;
 762         priv->var = mlx5_glue->dv_alloc_var(ctx, 0);
 763         if (!priv->var) {
 764                 DRV_LOG(ERR, "Failed to allocate VAR %u.\n", errno);
 765                 goto error;
 766         }
 767         priv->vdev = rte_vdpa_register_device(&pci_dev->device,
 768                         &mlx5_vdpa_ops);
 769         if (priv->vdev == NULL) {
 770                 DRV_LOG(ERR, "Failed to register vDPA device.");
 771                 rte_errno = rte_errno ? rte_errno : EINVAL;
 772                 goto error;
 773         }
 774         mlx5_vdpa_config_get(pci_dev->device.devargs, priv);
 775         SLIST_INIT(&priv->mr_list);
 776         pthread_mutex_lock(&priv_list_lock);
 777         TAILQ_INSERT_TAIL(&priv_list, priv, next);
 778         pthread_mutex_unlock(&priv_list_lock);
 779         return 0;
 780
 781 error:
 782         if (priv) {
 783                 if (priv->var)
 784                         mlx5_glue->dv_free_var(priv->var);
 785                 rte_free(priv);
 786         }
 787         if (ctx)
 788                 mlx5_glue->close_device(ctx);
 789         return -rte_errno;
 790 }
 791
 792 /**
 793  * DPDK callback to remove a PCI device.
 794  *
 795  * This function removes all vDPA devices belong to a given PCI device.
 796  *
 797  * @param[in] pci_dev
 798  *   Pointer to the PCI device.
 799  *
 800  * @return
 801  *   0 on success, the function cannot fail.
 802  */
 803 static int
 804 mlx5_vdpa_pci_remove(struct rte_pci_device *pci_dev)
 805 {
 806         struct mlx5_vdpa_priv *priv = NULL;
 807         int found = 0;
 808
 809         pthread_mutex_lock(&priv_list_lock);
 810         TAILQ_FOREACH(priv, &priv_list, next) {
 811                 if (!rte_pci_addr_cmp(&priv->pci_dev->addr, &pci_dev->addr)) {
 812                         found = 1;
 813                         break;
 814                 }
 815         }
 816         if (found)
 817                 TAILQ_REMOVE(&priv_list, priv, next);
 818         pthread_mutex_unlock(&priv_list_lock);
 819         if (found) {
 820                 if (priv->configured)
 821                         mlx5_vdpa_dev_close(priv->vid);
 822                 if (priv->var) {
 823                         mlx5_glue->dv_free_var(priv->var);
 824                         priv->var = NULL;
 825                 }
 826                 mlx5_glue->close_device(priv->ctx);
 827                 rte_free(priv);
 828         }
 829         return 0;
 830 }
 831
 832 static const struct rte_pci_id mlx5_vdpa_pci_id_map[] = {
 833         {
 834                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 835                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 836         },
 837         {
 838                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 839                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF)
 840         },
 841         {
 842                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 843                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
 844         },
 845         {
 846                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 847                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF)
 848         },
 849         {
 850                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 851                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
 852         },
 853         {
 854                 .vendor_id = 0
 855         }
 856 };
 857
 858 static struct rte_pci_driver mlx5_vdpa_driver = {
 859         .driver = {
 860                 .name = "mlx5_vdpa",
 861         },
 862         .id_table = mlx5_vdpa_pci_id_map,
 863         .probe = mlx5_vdpa_pci_probe,
 864         .remove = mlx5_vdpa_pci_remove,
 865         .drv_flags = 0,
 866 };
 867
 868 /**
 869  * Driver initialization routine.
 870  */
 871 RTE_INIT(rte_mlx5_vdpa_init)
 872 {
 873         /* Initialize common log type. */
 874         mlx5_vdpa_logtype = rte_log_register("pmd.vdpa.mlx5");
 875         if (mlx5_vdpa_logtype >= 0)
 876                 rte_log_set_level(mlx5_vdpa_logtype, RTE_LOG_NOTICE);
 877         if (mlx5_glue)
 878                 rte_pci_register(&mlx5_vdpa_driver);
 879 }
 880
 881 RTE_PMD_EXPORT_NAME(net_mlx5_vdpa, __COUNTER__);
 882 RTE_PMD_REGISTER_PCI_TABLE(net_mlx5_vdpa, mlx5_vdpa_pci_id_map);
 883 RTE_PMD_REGISTER_KMOD_DEP(net_mlx5_vdpa, "* ib_uverbs & mlx5_core & mlx5_ib");