drivers/vdpa/mlx5/mlx5_vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <unistd.h>
   5 #include <net/if.h>
   6 #include <sys/socket.h>
   7 #include <sys/ioctl.h>
   8 #include <fcntl.h>
   9 #include <netinet/in.h>
  10
  11 #include <rte_malloc.h>
  12 #include <rte_log.h>
  13 #include <rte_errno.h>
  14 #include <rte_pci.h>
  15 #include <rte_string_fns.h>
  16
  17 #include <mlx5_glue.h>
  18 #include <mlx5_common.h>
  19 #include <mlx5_common_pci.h>
  20 #include <mlx5_devx_cmds.h>
  21 #include <mlx5_prm.h>
  22 #include <mlx5_nl.h>
  23
  24 #include "mlx5_vdpa_utils.h"
  25 #include "mlx5_vdpa.h"
  26
  27
  28 #define MLX5_VDPA_DEFAULT_FEATURES ((1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
  29                             (1ULL << VIRTIO_F_ANY_LAYOUT) | \
  30                             (1ULL << VIRTIO_NET_F_MQ) | \
  31                             (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \
  32                             (1ULL << VIRTIO_F_ORDER_PLATFORM) | \
  33                             (1ULL << VHOST_F_LOG_ALL) | \
  34                             (1ULL << VIRTIO_NET_F_MTU))
  35
  36 #define MLX5_VDPA_PROTOCOL_FEATURES \
  37                             ((1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
  38                              (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
  39                              (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
  40                              (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) | \
  41                              (1ULL << VHOST_USER_PROTOCOL_F_MQ) | \
  42                              (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU) | \
  43                              (1ULL << VHOST_USER_PROTOCOL_F_STATUS))
  44
  45 #define MLX5_VDPA_MAX_RETRIES 20
  46 #define MLX5_VDPA_USEC 1000
  47 #define MLX5_VDPA_DEFAULT_NO_TRAFFIC_TIME_S 2LLU
  48
  49 TAILQ_HEAD(mlx5_vdpa_privs, mlx5_vdpa_priv) priv_list =
  50                                               TAILQ_HEAD_INITIALIZER(priv_list);
  51 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
  52
  53 static struct mlx5_vdpa_priv *
  54 mlx5_vdpa_find_priv_resource_by_vdev(struct rte_vdpa_device *vdev)
  55 {
  56         struct mlx5_vdpa_priv *priv;
  57         int found = 0;
  58
  59         pthread_mutex_lock(&priv_list_lock);
  60         TAILQ_FOREACH(priv, &priv_list, next) {
  61                 if (vdev == priv->vdev) {
  62                         found = 1;
  63                         break;
  64                 }
  65         }
  66         pthread_mutex_unlock(&priv_list_lock);
  67         if (!found) {
  68                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
  69                 rte_errno = EINVAL;
  70                 return NULL;
  71         }
  72         return priv;
  73 }
  74
  75 static int
  76 mlx5_vdpa_get_queue_num(struct rte_vdpa_device *vdev, uint32_t *queue_num)
  77 {
  78         struct mlx5_vdpa_priv *priv =
  79                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
  80
  81         if (priv == NULL) {
  82                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
  83                 return -1;
  84         }
  85         *queue_num = priv->caps.max_num_virtio_queues;
  86         return 0;
  87 }
  88
  89 static int
  90 mlx5_vdpa_get_vdpa_features(struct rte_vdpa_device *vdev, uint64_t *features)
  91 {
  92         struct mlx5_vdpa_priv *priv =
  93                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
  94
  95         if (priv == NULL) {
  96                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
  97                 return -1;
  98         }
  99         *features = MLX5_VDPA_DEFAULT_FEATURES;
 100         if (priv->caps.virtio_queue_type & (1 << MLX5_VIRTQ_TYPE_PACKED))
 101                 *features |= (1ULL << VIRTIO_F_RING_PACKED);
 102         if (priv->caps.tso_ipv4)
 103                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO4);
 104         if (priv->caps.tso_ipv6)
 105                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO6);
 106         if (priv->caps.tx_csum)
 107                 *features |= (1ULL << VIRTIO_NET_F_CSUM);
 108         if (priv->caps.rx_csum)
 109                 *features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
 110         if (priv->caps.virtio_version_1_0)
 111                 *features |= (1ULL << VIRTIO_F_VERSION_1);
 112         return 0;
 113 }
 114
 115 static int
 116 mlx5_vdpa_get_protocol_features(struct rte_vdpa_device *vdev,
 117                 uint64_t *features)
 118 {
 119         struct mlx5_vdpa_priv *priv =
 120                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 121
 122         if (priv == NULL) {
 123                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 124                 return -1;
 125         }
 126         *features = MLX5_VDPA_PROTOCOL_FEATURES;
 127         return 0;
 128 }
 129
 130 static int
 131 mlx5_vdpa_set_vring_state(int vid, int vring, int state)
 132 {
 133         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 134         struct mlx5_vdpa_priv *priv =
 135                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 136
 137         if (priv == NULL) {
 138                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 139                 return -EINVAL;
 140         }
 141         if (vring >= (int)priv->caps.max_num_virtio_queues * 2) {
 142                 DRV_LOG(ERR, "Too big vring id: %d.", vring);
 143                 return -E2BIG;
 144         }
 145         return mlx5_vdpa_virtq_enable(priv, vring, state);
 146 }
 147
 148 static int
 149 mlx5_vdpa_features_set(int vid)
 150 {
 151         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 152         struct mlx5_vdpa_priv *priv =
 153                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 154         uint64_t log_base, log_size;
 155         uint64_t features;
 156         int ret;
 157
 158         if (priv == NULL) {
 159                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 160                 return -EINVAL;
 161         }
 162         ret = rte_vhost_get_negotiated_features(vid, &features);
 163         if (ret) {
 164                 DRV_LOG(ERR, "Failed to get negotiated features.");
 165                 return ret;
 166         }
 167         if (RTE_VHOST_NEED_LOG(features)) {
 168                 ret = rte_vhost_get_log_base(vid, &log_base, &log_size);
 169                 if (ret) {
 170                         DRV_LOG(ERR, "Failed to get log base.");
 171                         return ret;
 172                 }
 173                 ret = mlx5_vdpa_dirty_bitmap_set(priv, log_base, log_size);
 174                 if (ret) {
 175                         DRV_LOG(ERR, "Failed to set dirty bitmap.");
 176                         return ret;
 177                 }
 178                 DRV_LOG(INFO, "mlx5 vdpa: enabling dirty logging...");
 179                 ret = mlx5_vdpa_logging_enable(priv, 1);
 180                 if (ret) {
 181                         DRV_LOG(ERR, "Failed t enable dirty logging.");
 182                         return ret;
 183                 }
 184         }
 185         return 0;
 186 }
 187
 188 static int
 189 mlx5_vdpa_pd_create(struct mlx5_vdpa_priv *priv)
 190 {
 191 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 192         priv->pd = mlx5_glue->alloc_pd(priv->ctx);
 193         if (priv->pd == NULL) {
 194                 DRV_LOG(ERR, "Failed to allocate PD.");
 195                 return errno ? -errno : -ENOMEM;
 196         }
 197         struct mlx5dv_obj obj;
 198         struct mlx5dv_pd pd_info;
 199         int ret = 0;
 200
 201         obj.pd.in = priv->pd;
 202         obj.pd.out = &pd_info;
 203         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
 204         if (ret) {
 205                 DRV_LOG(ERR, "Fail to get PD object info.");
 206                 mlx5_glue->dealloc_pd(priv->pd);
 207                 priv->pd = NULL;
 208                 return -errno;
 209         }
 210         priv->pdn = pd_info.pdn;
 211         return 0;
 212 #else
 213         (void)priv;
 214         DRV_LOG(ERR, "Cannot get pdn - no DV support.");
 215         return -ENOTSUP;
 216 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
 217 }
 218
 219 static int
 220 mlx5_vdpa_mtu_set(struct mlx5_vdpa_priv *priv)
 221 {
 222         struct ifreq request;
 223         uint16_t vhost_mtu = 0;
 224         uint16_t kern_mtu = 0;
 225         int ret = rte_vhost_get_mtu(priv->vid, &vhost_mtu);
 226         int sock;
 227         int retries = MLX5_VDPA_MAX_RETRIES;
 228
 229         if (ret) {
 230                 DRV_LOG(DEBUG, "Cannot get vhost MTU - %d.", ret);
 231                 return ret;
 232         }
 233         if (!vhost_mtu) {
 234                 DRV_LOG(DEBUG, "Vhost MTU is 0.");
 235                 return ret;
 236         }
 237         ret = mlx5_get_ifname_sysfs(priv->ctx->device->ibdev_path,
 238                                     request.ifr_name);
 239         if (ret) {
 240                 DRV_LOG(DEBUG, "Cannot get kernel IF name - %d.", ret);
 241                 return ret;
 242         }
 243         sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
 244         if (sock == -1) {
 245                 DRV_LOG(DEBUG, "Cannot open IF socket.");
 246                 return sock;
 247         }
 248         while (retries--) {
 249                 ret = ioctl(sock, SIOCGIFMTU, &request);
 250                 if (ret == -1)
 251                         break;
 252                 kern_mtu = request.ifr_mtu;
 253                 DRV_LOG(DEBUG, "MTU: current %d requested %d.", (int)kern_mtu,
 254                         (int)vhost_mtu);
 255                 if (kern_mtu == vhost_mtu)
 256                         break;
 257                 request.ifr_mtu = vhost_mtu;
 258                 ret = ioctl(sock, SIOCSIFMTU, &request);
 259                 if (ret == -1)
 260                         break;
 261                 request.ifr_mtu = 0;
 262                 usleep(MLX5_VDPA_USEC);
 263         }
 264         close(sock);
 265         return kern_mtu == vhost_mtu ? 0 : -1;
 266 }
 267
 268 static int
 269 mlx5_vdpa_dev_close(int vid)
 270 {
 271         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 272         struct mlx5_vdpa_priv *priv =
 273                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 274         int ret = 0;
 275
 276         if (priv == NULL) {
 277                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 278                 return -1;
 279         }
 280         if (priv->configured)
 281                 ret |= mlx5_vdpa_lm_log(priv);
 282         mlx5_vdpa_cqe_event_unset(priv);
 283         mlx5_vdpa_steer_unset(priv);
 284         mlx5_vdpa_virtqs_release(priv);
 285         mlx5_vdpa_event_qp_global_release(priv);
 286         mlx5_vdpa_mem_dereg(priv);
 287         if (priv->pd) {
 288                 claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 289                 priv->pd = NULL;
 290         }
 291         priv->configured = 0;
 292         priv->vid = 0;
 293         DRV_LOG(INFO, "vDPA device %d was closed.", vid);
 294         return ret;
 295 }
 296
 297 static int
 298 mlx5_vdpa_dev_config(int vid)
 299 {
 300         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 301         struct mlx5_vdpa_priv *priv =
 302                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 303
 304         if (priv == NULL) {
 305                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 306                 return -EINVAL;
 307         }
 308         if (priv->configured && mlx5_vdpa_dev_close(vid)) {
 309                 DRV_LOG(ERR, "Failed to reconfigure vid %d.", vid);
 310                 return -1;
 311         }
 312         priv->vid = vid;
 313         if (mlx5_vdpa_mtu_set(priv))
 314                 DRV_LOG(WARNING, "MTU cannot be set on device %s.",
 315                                 vdev->device->name);
 316         if (mlx5_vdpa_pd_create(priv) || mlx5_vdpa_mem_register(priv) ||
 317             mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) ||
 318             mlx5_vdpa_cqe_event_setup(priv)) {
 319                 mlx5_vdpa_dev_close(vid);
 320                 return -1;
 321         }
 322         priv->configured = 1;
 323         DRV_LOG(INFO, "vDPA device %d was configured.", vid);
 324         return 0;
 325 }
 326
 327 static int
 328 mlx5_vdpa_get_device_fd(int vid)
 329 {
 330         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 331         struct mlx5_vdpa_priv *priv =
 332                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 333
 334         if (priv == NULL) {
 335                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 336                 return -EINVAL;
 337         }
 338         return priv->ctx->cmd_fd;
 339 }
 340
 341 static int
 342 mlx5_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
 343 {
 344         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
 345         struct mlx5_vdpa_priv *priv =
 346                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 347
 348         RTE_SET_USED(qid);
 349         if (priv == NULL) {
 350                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 351                 return -EINVAL;
 352         }
 353         if (!priv->var) {
 354                 DRV_LOG(ERR, "VAR was not created for device %s, is the device"
 355                         " configured?.", vdev->device->name);
 356                 return -EINVAL;
 357         }
 358         *offset = priv->var->mmap_off;
 359         *size = priv->var->length;
 360         return 0;
 361 }
 362
 363 static int
 364 mlx5_vdpa_get_stats_names(struct rte_vdpa_device *vdev,
 365                 struct rte_vdpa_stat_name *stats_names,
 366                 unsigned int size)
 367 {
 368         static const char *mlx5_vdpa_stats_names[MLX5_VDPA_STATS_MAX] = {
 369                 "received_descriptors",
 370                 "completed_descriptors",
 371                 "bad descriptor errors",
 372                 "exceed max chain",
 373                 "invalid buffer",
 374                 "completion errors",
 375         };
 376         struct mlx5_vdpa_priv *priv =
 377                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 378         unsigned int i;
 379
 380         if (priv == NULL) {
 381                 DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
 382                 return -ENODEV;
 383         }
 384         if (!stats_names)
 385                 return MLX5_VDPA_STATS_MAX;
 386         size = RTE_MIN(size, (unsigned int)MLX5_VDPA_STATS_MAX);
 387         for (i = 0; i < size; ++i)
 388                 strlcpy(stats_names[i].name, mlx5_vdpa_stats_names[i],
 389                         RTE_VDPA_STATS_NAME_SIZE);
 390         return size;
 391 }
 392
 393 static int
 394 mlx5_vdpa_get_stats(struct rte_vdpa_device *vdev, int qid,
 395                 struct rte_vdpa_stat *stats, unsigned int n)
 396 {
 397         struct mlx5_vdpa_priv *priv =
 398                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 399
 400         if (priv == NULL) {
 401                 DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
 402                 return -ENODEV;
 403         }
 404         if (!priv->configured) {
 405                 DRV_LOG(ERR, "Device %s was not configured.",
 406                                 vdev->device->name);
 407                 return -ENODATA;
 408         }
 409         if (qid >= (int)priv->nr_virtqs) {
 410                 DRV_LOG(ERR, "Too big vring id: %d for device %s.", qid,
 411                                 vdev->device->name);
 412                 return -E2BIG;
 413         }
 414         if (!priv->caps.queue_counters_valid) {
 415                 DRV_LOG(ERR, "Virtq statistics is not supported for device %s.",
 416                         vdev->device->name);
 417                 return -ENOTSUP;
 418         }
 419         return mlx5_vdpa_virtq_stats_get(priv, qid, stats, n);
 420 }
 421
 422 static int
 423 mlx5_vdpa_reset_stats(struct rte_vdpa_device *vdev, int qid)
 424 {
 425         struct mlx5_vdpa_priv *priv =
 426                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 427
 428         if (priv == NULL) {
 429                 DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
 430                 return -ENODEV;
 431         }
 432         if (!priv->configured) {
 433                 DRV_LOG(ERR, "Device %s was not configured.",
 434                                 vdev->device->name);
 435                 return -ENODATA;
 436         }
 437         if (qid >= (int)priv->nr_virtqs) {
 438                 DRV_LOG(ERR, "Too big vring id: %d for device %s.", qid,
 439                                 vdev->device->name);
 440                 return -E2BIG;
 441         }
 442         if (!priv->caps.queue_counters_valid) {
 443                 DRV_LOG(ERR, "Virtq statistics is not supported for device %s.",
 444                         vdev->device->name);
 445                 return -ENOTSUP;
 446         }
 447         return mlx5_vdpa_virtq_stats_reset(priv, qid);
 448 }
 449
 450 static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {
 451         .get_queue_num = mlx5_vdpa_get_queue_num,
 452         .get_features = mlx5_vdpa_get_vdpa_features,
 453         .get_protocol_features = mlx5_vdpa_get_protocol_features,
 454         .dev_conf = mlx5_vdpa_dev_config,
 455         .dev_close = mlx5_vdpa_dev_close,
 456         .set_vring_state = mlx5_vdpa_set_vring_state,
 457         .set_features = mlx5_vdpa_features_set,
 458         .migration_done = NULL,
 459         .get_vfio_group_fd = NULL,
 460         .get_vfio_device_fd = mlx5_vdpa_get_device_fd,
 461         .get_notify_area = mlx5_vdpa_get_notify_area,
 462         .get_stats_names = mlx5_vdpa_get_stats_names,
 463         .get_stats = mlx5_vdpa_get_stats,
 464         .reset_stats = mlx5_vdpa_reset_stats,
 465 };
 466
 467 static struct ibv_device *
 468 mlx5_vdpa_get_ib_device_match(struct rte_pci_addr *addr)
 469 {
 470         int n;
 471         struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
 472         struct ibv_device *ibv_match = NULL;
 473
 474         if (!ibv_list) {
 475                 rte_errno = ENOSYS;
 476                 return NULL;
 477         }
 478         while (n-- > 0) {
 479                 struct rte_pci_addr pci_addr;
 480
 481                 DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
 482                 if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &pci_addr))
 483                         continue;
 484                 if (rte_pci_addr_cmp(addr, &pci_addr))
 485                         continue;
 486                 ibv_match = ibv_list[n];
 487                 break;
 488         }
 489         if (!ibv_match)
 490                 rte_errno = ENOENT;
 491         mlx5_glue->free_device_list(ibv_list);
 492         return ibv_match;
 493 }
 494
 495 /* Try to disable ROCE by Netlink\Devlink. */
 496 static int
 497 mlx5_vdpa_nl_roce_disable(const char *addr)
 498 {
 499         int nlsk_fd = mlx5_nl_init(NETLINK_GENERIC);
 500         int devlink_id;
 501         int enable;
 502         int ret;
 503
 504         if (nlsk_fd < 0)
 505                 return nlsk_fd;
 506         devlink_id = mlx5_nl_devlink_family_id_get(nlsk_fd);
 507         if (devlink_id < 0) {
 508                 ret = devlink_id;
 509                 DRV_LOG(DEBUG, "Failed to get devlink id for ROCE operations by"
 510                         " Netlink.");
 511                 goto close;
 512         }
 513         ret = mlx5_nl_enable_roce_get(nlsk_fd, devlink_id, addr, &enable);
 514         if (ret) {
 515                 DRV_LOG(DEBUG, "Failed to get ROCE enable by Netlink: %d.",
 516                         ret);
 517                 goto close;
 518         } else if (!enable) {
 519                 DRV_LOG(INFO, "ROCE has already disabled(Netlink).");
 520                 goto close;
 521         }
 522         ret = mlx5_nl_enable_roce_set(nlsk_fd, devlink_id, addr, 0);
 523         if (ret)
 524                 DRV_LOG(DEBUG, "Failed to disable ROCE by Netlink: %d.", ret);
 525         else
 526                 DRV_LOG(INFO, "ROCE is disabled by Netlink successfully.");
 527 close:
 528         close(nlsk_fd);
 529         return ret;
 530 }
 531
 532 /* Try to disable ROCE by sysfs. */
 533 static int
 534 mlx5_vdpa_sys_roce_disable(const char *addr)
 535 {
 536         FILE *file_o;
 537         int enable;
 538         int ret;
 539
 540         MKSTR(file_p, "/sys/bus/pci/devices/%s/roce_enable", addr);
 541         file_o = fopen(file_p, "rb");
 542         if (!file_o) {
 543                 rte_errno = ENOTSUP;
 544                 return -ENOTSUP;
 545         }
 546         ret = fscanf(file_o, "%d", &enable);
 547         if (ret != 1) {
 548                 rte_errno = EINVAL;
 549                 ret = EINVAL;
 550                 goto close;
 551         } else if (!enable) {
 552                 ret = 0;
 553                 DRV_LOG(INFO, "ROCE has already disabled(sysfs).");
 554                 goto close;
 555         }
 556         fclose(file_o);
 557         file_o = fopen(file_p, "wb");
 558         if (!file_o) {
 559                 rte_errno = ENOTSUP;
 560                 return -ENOTSUP;
 561         }
 562         fprintf(file_o, "0\n");
 563         ret = 0;
 564 close:
 565         if (ret)
 566                 DRV_LOG(DEBUG, "Failed to disable ROCE by sysfs: %d.", ret);
 567         else
 568                 DRV_LOG(INFO, "ROCE is disabled by sysfs successfully.");
 569         fclose(file_o);
 570         return ret;
 571 }
 572
 573 static int
 574 mlx5_vdpa_roce_disable(struct rte_pci_addr *addr, struct ibv_device **ibv)
 575 {
 576         char addr_name[64] = {0};
 577
 578         rte_pci_device_name(addr, addr_name, sizeof(addr_name));
 579         /* Firstly try to disable ROCE by Netlink and fallback to sysfs. */
 580         if (mlx5_vdpa_nl_roce_disable(addr_name) == 0 ||
 581             mlx5_vdpa_sys_roce_disable(addr_name) == 0) {
 582                 /*
 583                  * Succeed to disable ROCE, wait for the IB device to appear
 584                  * again after reload.
 585                  */
 586                 int r;
 587                 struct ibv_device *ibv_new;
 588
 589                 for (r = MLX5_VDPA_MAX_RETRIES; r; r--) {
 590                         ibv_new = mlx5_vdpa_get_ib_device_match(addr);
 591                         if (ibv_new) {
 592                                 *ibv = ibv_new;
 593                                 return 0;
 594                         }
 595                         usleep(MLX5_VDPA_USEC);
 596                 }
 597                 DRV_LOG(ERR, "Cannot much device %s after ROCE disable, "
 598                         "retries exceed %d", addr_name, MLX5_VDPA_MAX_RETRIES);
 599                 rte_errno = EAGAIN;
 600         }
 601         return -rte_errno;
 602 }
 603
 604 static int
 605 mlx5_vdpa_args_check_handler(const char *key, const char *val, void *opaque)
 606 {
 607         struct mlx5_vdpa_priv *priv = opaque;
 608         unsigned long tmp;
 609
 610         if (strcmp(key, "class") == 0)
 611                 return 0;
 612         errno = 0;
 613         tmp = strtoul(val, NULL, 0);
 614         if (errno) {
 615                 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 616                 return -errno;
 617         }
 618         if (strcmp(key, "event_mode") == 0) {
 619                 if (tmp <= MLX5_VDPA_EVENT_MODE_ONLY_INTERRUPT)
 620                         priv->event_mode = (int)tmp;
 621                 else
 622                         DRV_LOG(WARNING, "Invalid event_mode %s.", val);
 623         } else if (strcmp(key, "event_us") == 0) {
 624                 priv->event_us = (uint32_t)tmp;
 625         } else if (strcmp(key, "no_traffic_time") == 0) {
 626                 priv->no_traffic_time_s = (uint32_t)tmp;
 627         } else {
 628                 DRV_LOG(WARNING, "Invalid key %s.", key);
 629         }
 630         return 0;
 631 }
 632
 633 static void
 634 mlx5_vdpa_config_get(struct rte_devargs *devargs, struct mlx5_vdpa_priv *priv)
 635 {
 636         struct rte_kvargs *kvlist;
 637
 638         priv->event_mode = MLX5_VDPA_EVENT_MODE_DYNAMIC_TIMER;
 639         priv->event_us = 0;
 640         priv->no_traffic_time_s = MLX5_VDPA_DEFAULT_NO_TRAFFIC_TIME_S;
 641         if (devargs == NULL)
 642                 return;
 643         kvlist = rte_kvargs_parse(devargs->args, NULL);
 644         if (kvlist == NULL)
 645                 return;
 646         rte_kvargs_process(kvlist, NULL, mlx5_vdpa_args_check_handler, priv);
 647         rte_kvargs_free(kvlist);
 648         if (!priv->event_us) {
 649                 if (priv->event_mode == MLX5_VDPA_EVENT_MODE_DYNAMIC_TIMER)
 650                         priv->event_us = MLX5_VDPA_DEFAULT_TIMER_STEP_US;
 651                 else if (priv->event_mode == MLX5_VDPA_EVENT_MODE_FIXED_TIMER)
 652                         priv->event_us = MLX5_VDPA_DEFAULT_TIMER_DELAY_US;
 653         }
 654         DRV_LOG(DEBUG, "event mode is %d.", priv->event_mode);
 655         DRV_LOG(DEBUG, "event_us is %u us.", priv->event_us);
 656         DRV_LOG(DEBUG, "no traffic time is %u s.", priv->no_traffic_time_s);
 657 }
 658
 659 /**
 660  * DPDK callback to register a mlx5 PCI device.
 661  *
 662  * This function spawns vdpa device out of a given PCI device.
 663  *
 664  * @param[in] pci_drv
 665  *   PCI driver structure (mlx5_vpda_driver).
 666  * @param[in] pci_dev
 667  *   PCI device information.
 668  *
 669  * @return
 670  *   0 on success, 1 to skip this driver, a negative errno value otherwise
 671  *   and rte_errno is set.
 672  */
 673 static int
 674 mlx5_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 675                     struct rte_pci_device *pci_dev __rte_unused)
 676 {
 677         struct ibv_device *ibv;
 678         struct mlx5_vdpa_priv *priv = NULL;
 679         struct ibv_context *ctx = NULL;
 680         struct mlx5_hca_attr attr;
 681         int ret;
 682
 683         if (mlx5_class_get(pci_dev->device.devargs) != MLX5_CLASS_VDPA) {
 684                 DRV_LOG(DEBUG, "Skip probing - should be probed by other mlx5"
 685                         " driver.");
 686                 return 1;
 687         }
 688         ibv = mlx5_vdpa_get_ib_device_match(&pci_dev->addr);
 689         if (!ibv) {
 690                 DRV_LOG(ERR, "No matching IB device for PCI slot "
 691                         PCI_PRI_FMT ".", pci_dev->addr.domain,
 692                         pci_dev->addr.bus, pci_dev->addr.devid,
 693                         pci_dev->addr.function);
 694                 return -rte_errno;
 695         } else {
 696                 DRV_LOG(INFO, "PCI information matches for device \"%s\".",
 697                         ibv->name);
 698         }
 699         if (mlx5_vdpa_roce_disable(&pci_dev->addr, &ibv) != 0) {
 700                 DRV_LOG(WARNING, "Failed to disable ROCE for \"%s\".",
 701                         ibv->name);
 702                 return -rte_errno;
 703         }
 704         ctx = mlx5_glue->dv_open_device(ibv);
 705         if (!ctx) {
 706                 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
 707                 rte_errno = ENODEV;
 708                 return -rte_errno;
 709         }
 710         ret = mlx5_devx_cmd_query_hca_attr(ctx, &attr);
 711         if (ret) {
 712                 DRV_LOG(ERR, "Unable to read HCA capabilities.");
 713                 rte_errno = ENOTSUP;
 714                 goto error;
 715         } else if (!attr.vdpa.valid || !attr.vdpa.max_num_virtio_queues) {
 716                 DRV_LOG(ERR, "Not enough capabilities to support vdpa, maybe "
 717                         "old FW/OFED version?");
 718                 rte_errno = ENOTSUP;
 719                 goto error;
 720         }
 721         if (!attr.vdpa.queue_counters_valid)
 722                 DRV_LOG(DEBUG, "No capability to support virtq statistics.");
 723         priv = rte_zmalloc("mlx5 vDPA device private", sizeof(*priv) +
 724                            sizeof(struct mlx5_vdpa_virtq) *
 725                            attr.vdpa.max_num_virtio_queues * 2,
 726                            RTE_CACHE_LINE_SIZE);
 727         if (!priv) {
 728                 DRV_LOG(ERR, "Failed to allocate private memory.");
 729                 rte_errno = ENOMEM;
 730                 goto error;
 731         }
 732         priv->caps = attr.vdpa;
 733         priv->log_max_rqt_size = attr.log_max_rqt_size;
 734         priv->ctx = ctx;
 735         priv->pci_dev = pci_dev;
 736         priv->var = mlx5_glue->dv_alloc_var(ctx, 0);
 737         if (!priv->var) {
 738                 DRV_LOG(ERR, "Failed to allocate VAR %u.\n", errno);
 739                 goto error;
 740         }
 741         priv->vdev = rte_vdpa_register_device(&pci_dev->device,
 742                         &mlx5_vdpa_ops);
 743         if (priv->vdev == NULL) {
 744                 DRV_LOG(ERR, "Failed to register vDPA device.");
 745                 rte_errno = rte_errno ? rte_errno : EINVAL;
 746                 goto error;
 747         }
 748         mlx5_vdpa_config_get(pci_dev->device.devargs, priv);
 749         SLIST_INIT(&priv->mr_list);
 750         pthread_mutex_lock(&priv_list_lock);
 751         TAILQ_INSERT_TAIL(&priv_list, priv, next);
 752         pthread_mutex_unlock(&priv_list_lock);
 753         return 0;
 754
 755 error:
 756         if (priv) {
 757                 if (priv->var)
 758                         mlx5_glue->dv_free_var(priv->var);
 759                 rte_free(priv);
 760         }
 761         if (ctx)
 762                 mlx5_glue->close_device(ctx);
 763         return -rte_errno;
 764 }
 765
 766 /**
 767  * DPDK callback to remove a PCI device.
 768  *
 769  * This function removes all vDPA devices belong to a given PCI device.
 770  *
 771  * @param[in] pci_dev
 772  *   Pointer to the PCI device.
 773  *
 774  * @return
 775  *   0 on success, the function cannot fail.
 776  */
 777 static int
 778 mlx5_vdpa_pci_remove(struct rte_pci_device *pci_dev)
 779 {
 780         struct mlx5_vdpa_priv *priv = NULL;
 781         int found = 0;
 782
 783         pthread_mutex_lock(&priv_list_lock);
 784         TAILQ_FOREACH(priv, &priv_list, next) {
 785                 if (!rte_pci_addr_cmp(&priv->pci_dev->addr, &pci_dev->addr)) {
 786                         found = 1;
 787                         break;
 788                 }
 789         }
 790         if (found)
 791                 TAILQ_REMOVE(&priv_list, priv, next);
 792         pthread_mutex_unlock(&priv_list_lock);
 793         if (found) {
 794                 if (priv->configured)
 795                         mlx5_vdpa_dev_close(priv->vid);
 796                 if (priv->var) {
 797                         mlx5_glue->dv_free_var(priv->var);
 798                         priv->var = NULL;
 799                 }
 800                 mlx5_glue->close_device(priv->ctx);
 801                 rte_free(priv);
 802         }
 803         return 0;
 804 }
 805
 806 static const struct rte_pci_id mlx5_vdpa_pci_id_map[] = {
 807         {
 808                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 809                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 810         },
 811         {
 812                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 813                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF)
 814         },
 815         {
 816                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 817                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
 818         },
 819         {
 820                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 821                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF)
 822         },
 823         {
 824                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 825                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
 826         },
 827         {
 828                 .vendor_id = 0
 829         }
 830 };
 831
 832 static struct mlx5_pci_driver mlx5_vdpa_driver = {
 833         .driver_class = MLX5_CLASS_VDPA,
 834         .pci_driver = {
 835                 .driver = {
 836                         .name = "mlx5_vdpa",
 837                 },
 838                 .id_table = mlx5_vdpa_pci_id_map,
 839                 .probe = mlx5_vdpa_pci_probe,
 840                 .remove = mlx5_vdpa_pci_remove,
 841                 .drv_flags = 0,
 842         },
 843 };
 844
 845 RTE_LOG_REGISTER(mlx5_vdpa_logtype, pmd.vdpa.mlx5, NOTICE)
 846
 847 /**
 848  * Driver initialization routine.
 849  */
 850 RTE_INIT(rte_mlx5_vdpa_init)
 851 {
 852         mlx5_common_init();
 853         if (mlx5_glue)
 854                 mlx5_pci_driver_register(&mlx5_vdpa_driver);
 855 }
 856
 857 RTE_PMD_EXPORT_NAME(net_mlx5_vdpa, __COUNTER__);
 858 RTE_PMD_REGISTER_PCI_TABLE(net_mlx5_vdpa, mlx5_vdpa_pci_id_map);
 859 RTE_PMD_REGISTER_KMOD_DEP(net_mlx5_vdpa, "* ib_uverbs & mlx5_core & mlx5_ib");