vdpa/mlx5: fix queue update synchronization
[dpdk.git] / drivers / vdpa / mlx5 / mlx5_vdpa.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright 2019 Mellanox Technologies, Ltd
3  */
4 #include <unistd.h>
5 #include <net/if.h>
6 #include <sys/socket.h>
7 #include <sys/ioctl.h>
8 #include <fcntl.h>
9 #include <netinet/in.h>
10
11 #include <rte_malloc.h>
12 #include <rte_log.h>
13 #include <rte_errno.h>
14 #include <rte_pci.h>
15 #include <rte_string_fns.h>
16
17 #include <mlx5_glue.h>
18 #include <mlx5_common.h>
19 #include <mlx5_common_pci.h>
20 #include <mlx5_devx_cmds.h>
21 #include <mlx5_prm.h>
22 #include <mlx5_nl.h>
23
24 #include "mlx5_vdpa_utils.h"
25 #include "mlx5_vdpa.h"
26
27
28 #define MLX5_VDPA_DEFAULT_FEATURES ((1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
29                             (1ULL << VIRTIO_F_ANY_LAYOUT) | \
30                             (1ULL << VIRTIO_NET_F_MQ) | \
31                             (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \
32                             (1ULL << VIRTIO_F_ORDER_PLATFORM) | \
33                             (1ULL << VHOST_F_LOG_ALL) | \
34                             (1ULL << VIRTIO_NET_F_MTU))
35
36 #define MLX5_VDPA_PROTOCOL_FEATURES \
37                             ((1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
38                              (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
39                              (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
40                              (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) | \
41                              (1ULL << VHOST_USER_PROTOCOL_F_MQ) | \
42                              (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU) | \
43                              (1ULL << VHOST_USER_PROTOCOL_F_STATUS))
44
45 #define MLX5_VDPA_MAX_RETRIES 20
46 #define MLX5_VDPA_USEC 1000
47 #define MLX5_VDPA_DEFAULT_NO_TRAFFIC_TIME_S 2LLU
48
49 TAILQ_HEAD(mlx5_vdpa_privs, mlx5_vdpa_priv) priv_list =
50                                               TAILQ_HEAD_INITIALIZER(priv_list);
51 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
52
53 static struct mlx5_vdpa_priv *
54 mlx5_vdpa_find_priv_resource_by_vdev(struct rte_vdpa_device *vdev)
55 {
56         struct mlx5_vdpa_priv *priv;
57         int found = 0;
58
59         pthread_mutex_lock(&priv_list_lock);
60         TAILQ_FOREACH(priv, &priv_list, next) {
61                 if (vdev == priv->vdev) {
62                         found = 1;
63                         break;
64                 }
65         }
66         pthread_mutex_unlock(&priv_list_lock);
67         if (!found) {
68                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
69                 rte_errno = EINVAL;
70                 return NULL;
71         }
72         return priv;
73 }
74
75 static int
76 mlx5_vdpa_get_queue_num(struct rte_vdpa_device *vdev, uint32_t *queue_num)
77 {
78         struct mlx5_vdpa_priv *priv =
79                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
80
81         if (priv == NULL) {
82                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
83                 return -1;
84         }
85         *queue_num = priv->caps.max_num_virtio_queues;
86         return 0;
87 }
88
89 static int
90 mlx5_vdpa_get_vdpa_features(struct rte_vdpa_device *vdev, uint64_t *features)
91 {
92         struct mlx5_vdpa_priv *priv =
93                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
94
95         if (priv == NULL) {
96                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
97                 return -1;
98         }
99         *features = MLX5_VDPA_DEFAULT_FEATURES;
100         if (priv->caps.virtio_queue_type & (1 << MLX5_VIRTQ_TYPE_PACKED))
101                 *features |= (1ULL << VIRTIO_F_RING_PACKED);
102         if (priv->caps.tso_ipv4)
103                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO4);
104         if (priv->caps.tso_ipv6)
105                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO6);
106         if (priv->caps.tx_csum)
107                 *features |= (1ULL << VIRTIO_NET_F_CSUM);
108         if (priv->caps.rx_csum)
109                 *features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
110         if (priv->caps.virtio_version_1_0)
111                 *features |= (1ULL << VIRTIO_F_VERSION_1);
112         return 0;
113 }
114
115 static int
116 mlx5_vdpa_get_protocol_features(struct rte_vdpa_device *vdev,
117                 uint64_t *features)
118 {
119         struct mlx5_vdpa_priv *priv =
120                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
121
122         if (priv == NULL) {
123                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
124                 return -1;
125         }
126         *features = MLX5_VDPA_PROTOCOL_FEATURES;
127         return 0;
128 }
129
130 static int
131 mlx5_vdpa_set_vring_state(int vid, int vring, int state)
132 {
133         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
134         struct mlx5_vdpa_priv *priv =
135                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
136         int ret;
137
138         if (priv == NULL) {
139                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
140                 return -EINVAL;
141         }
142         if (vring >= (int)priv->caps.max_num_virtio_queues * 2) {
143                 DRV_LOG(ERR, "Too big vring id: %d.", vring);
144                 return -E2BIG;
145         }
146         pthread_mutex_lock(&priv->vq_config_lock);
147         ret = mlx5_vdpa_virtq_enable(priv, vring, state);
148         pthread_mutex_unlock(&priv->vq_config_lock);
149         return ret;
150 }
151
152 static int
153 mlx5_vdpa_features_set(int vid)
154 {
155         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
156         struct mlx5_vdpa_priv *priv =
157                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
158         uint64_t log_base, log_size;
159         uint64_t features;
160         int ret;
161
162         if (priv == NULL) {
163                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
164                 return -EINVAL;
165         }
166         ret = rte_vhost_get_negotiated_features(vid, &features);
167         if (ret) {
168                 DRV_LOG(ERR, "Failed to get negotiated features.");
169                 return ret;
170         }
171         if (RTE_VHOST_NEED_LOG(features)) {
172                 ret = rte_vhost_get_log_base(vid, &log_base, &log_size);
173                 if (ret) {
174                         DRV_LOG(ERR, "Failed to get log base.");
175                         return ret;
176                 }
177                 ret = mlx5_vdpa_dirty_bitmap_set(priv, log_base, log_size);
178                 if (ret) {
179                         DRV_LOG(ERR, "Failed to set dirty bitmap.");
180                         return ret;
181                 }
182                 DRV_LOG(INFO, "mlx5 vdpa: enabling dirty logging...");
183                 ret = mlx5_vdpa_logging_enable(priv, 1);
184                 if (ret) {
185                         DRV_LOG(ERR, "Failed t enable dirty logging.");
186                         return ret;
187                 }
188         }
189         return 0;
190 }
191
192 static int
193 mlx5_vdpa_pd_create(struct mlx5_vdpa_priv *priv)
194 {
195 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
196         priv->pd = mlx5_glue->alloc_pd(priv->ctx);
197         if (priv->pd == NULL) {
198                 DRV_LOG(ERR, "Failed to allocate PD.");
199                 return errno ? -errno : -ENOMEM;
200         }
201         struct mlx5dv_obj obj;
202         struct mlx5dv_pd pd_info;
203         int ret = 0;
204
205         obj.pd.in = priv->pd;
206         obj.pd.out = &pd_info;
207         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
208         if (ret) {
209                 DRV_LOG(ERR, "Fail to get PD object info.");
210                 mlx5_glue->dealloc_pd(priv->pd);
211                 priv->pd = NULL;
212                 return -errno;
213         }
214         priv->pdn = pd_info.pdn;
215         return 0;
216 #else
217         (void)priv;
218         DRV_LOG(ERR, "Cannot get pdn - no DV support.");
219         return -ENOTSUP;
220 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
221 }
222
223 static int
224 mlx5_vdpa_mtu_set(struct mlx5_vdpa_priv *priv)
225 {
226         struct ifreq request;
227         uint16_t vhost_mtu = 0;
228         uint16_t kern_mtu = 0;
229         int ret = rte_vhost_get_mtu(priv->vid, &vhost_mtu);
230         int sock;
231         int retries = MLX5_VDPA_MAX_RETRIES;
232
233         if (ret) {
234                 DRV_LOG(DEBUG, "Cannot get vhost MTU - %d.", ret);
235                 return ret;
236         }
237         if (!vhost_mtu) {
238                 DRV_LOG(DEBUG, "Vhost MTU is 0.");
239                 return ret;
240         }
241         ret = mlx5_get_ifname_sysfs(priv->ctx->device->ibdev_path,
242                                     request.ifr_name);
243         if (ret) {
244                 DRV_LOG(DEBUG, "Cannot get kernel IF name - %d.", ret);
245                 return ret;
246         }
247         sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
248         if (sock == -1) {
249                 DRV_LOG(DEBUG, "Cannot open IF socket.");
250                 return sock;
251         }
252         while (retries--) {
253                 ret = ioctl(sock, SIOCGIFMTU, &request);
254                 if (ret == -1)
255                         break;
256                 kern_mtu = request.ifr_mtu;
257                 DRV_LOG(DEBUG, "MTU: current %d requested %d.", (int)kern_mtu,
258                         (int)vhost_mtu);
259                 if (kern_mtu == vhost_mtu)
260                         break;
261                 request.ifr_mtu = vhost_mtu;
262                 ret = ioctl(sock, SIOCSIFMTU, &request);
263                 if (ret == -1)
264                         break;
265                 request.ifr_mtu = 0;
266                 usleep(MLX5_VDPA_USEC);
267         }
268         close(sock);
269         return kern_mtu == vhost_mtu ? 0 : -1;
270 }
271
272 static int
273 mlx5_vdpa_dev_close(int vid)
274 {
275         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
276         struct mlx5_vdpa_priv *priv =
277                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
278         int ret = 0;
279
280         if (priv == NULL) {
281                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
282                 return -1;
283         }
284         if (priv->configured)
285                 ret |= mlx5_vdpa_lm_log(priv);
286         mlx5_vdpa_cqe_event_unset(priv);
287         mlx5_vdpa_steer_unset(priv);
288         mlx5_vdpa_virtqs_release(priv);
289         mlx5_vdpa_event_qp_global_release(priv);
290         mlx5_vdpa_mem_dereg(priv);
291         if (priv->pd) {
292                 claim_zero(mlx5_glue->dealloc_pd(priv->pd));
293                 priv->pd = NULL;
294         }
295         priv->configured = 0;
296         priv->vid = 0;
297         DRV_LOG(INFO, "vDPA device %d was closed.", vid);
298         return ret;
299 }
300
301 static int
302 mlx5_vdpa_dev_config(int vid)
303 {
304         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
305         struct mlx5_vdpa_priv *priv =
306                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
307
308         if (priv == NULL) {
309                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
310                 return -EINVAL;
311         }
312         if (priv->configured && mlx5_vdpa_dev_close(vid)) {
313                 DRV_LOG(ERR, "Failed to reconfigure vid %d.", vid);
314                 return -1;
315         }
316         priv->vid = vid;
317         if (mlx5_vdpa_mtu_set(priv))
318                 DRV_LOG(WARNING, "MTU cannot be set on device %s.",
319                                 vdev->device->name);
320         if (mlx5_vdpa_pd_create(priv) || mlx5_vdpa_mem_register(priv) ||
321             mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) ||
322             mlx5_vdpa_cqe_event_setup(priv)) {
323                 mlx5_vdpa_dev_close(vid);
324                 return -1;
325         }
326         priv->configured = 1;
327         DRV_LOG(INFO, "vDPA device %d was configured.", vid);
328         return 0;
329 }
330
331 static int
332 mlx5_vdpa_get_device_fd(int vid)
333 {
334         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
335         struct mlx5_vdpa_priv *priv =
336                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
337
338         if (priv == NULL) {
339                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
340                 return -EINVAL;
341         }
342         return priv->ctx->cmd_fd;
343 }
344
345 static int
346 mlx5_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
347 {
348         struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
349         struct mlx5_vdpa_priv *priv =
350                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
351
352         RTE_SET_USED(qid);
353         if (priv == NULL) {
354                 DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
355                 return -EINVAL;
356         }
357         if (!priv->var) {
358                 DRV_LOG(ERR, "VAR was not created for device %s, is the device"
359                         " configured?.", vdev->device->name);
360                 return -EINVAL;
361         }
362         *offset = priv->var->mmap_off;
363         *size = priv->var->length;
364         return 0;
365 }
366
367 static int
368 mlx5_vdpa_get_stats_names(struct rte_vdpa_device *vdev,
369                 struct rte_vdpa_stat_name *stats_names,
370                 unsigned int size)
371 {
372         static const char *mlx5_vdpa_stats_names[MLX5_VDPA_STATS_MAX] = {
373                 "received_descriptors",
374                 "completed_descriptors",
375                 "bad descriptor errors",
376                 "exceed max chain",
377                 "invalid buffer",
378                 "completion errors",
379         };
380         struct mlx5_vdpa_priv *priv =
381                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
382         unsigned int i;
383
384         if (priv == NULL) {
385                 DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
386                 return -ENODEV;
387         }
388         if (!stats_names)
389                 return MLX5_VDPA_STATS_MAX;
390         size = RTE_MIN(size, (unsigned int)MLX5_VDPA_STATS_MAX);
391         for (i = 0; i < size; ++i)
392                 strlcpy(stats_names[i].name, mlx5_vdpa_stats_names[i],
393                         RTE_VDPA_STATS_NAME_SIZE);
394         return size;
395 }
396
397 static int
398 mlx5_vdpa_get_stats(struct rte_vdpa_device *vdev, int qid,
399                 struct rte_vdpa_stat *stats, unsigned int n)
400 {
401         struct mlx5_vdpa_priv *priv =
402                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
403
404         if (priv == NULL) {
405                 DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
406                 return -ENODEV;
407         }
408         if (!priv->configured) {
409                 DRV_LOG(ERR, "Device %s was not configured.",
410                                 vdev->device->name);
411                 return -ENODATA;
412         }
413         if (qid >= (int)priv->nr_virtqs) {
414                 DRV_LOG(ERR, "Too big vring id: %d for device %s.", qid,
415                                 vdev->device->name);
416                 return -E2BIG;
417         }
418         if (!priv->caps.queue_counters_valid) {
419                 DRV_LOG(ERR, "Virtq statistics is not supported for device %s.",
420                         vdev->device->name);
421                 return -ENOTSUP;
422         }
423         return mlx5_vdpa_virtq_stats_get(priv, qid, stats, n);
424 }
425
426 static int
427 mlx5_vdpa_reset_stats(struct rte_vdpa_device *vdev, int qid)
428 {
429         struct mlx5_vdpa_priv *priv =
430                 mlx5_vdpa_find_priv_resource_by_vdev(vdev);
431
432         if (priv == NULL) {
433                 DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
434                 return -ENODEV;
435         }
436         if (!priv->configured) {
437                 DRV_LOG(ERR, "Device %s was not configured.",
438                                 vdev->device->name);
439                 return -ENODATA;
440         }
441         if (qid >= (int)priv->nr_virtqs) {
442                 DRV_LOG(ERR, "Too big vring id: %d for device %s.", qid,
443                                 vdev->device->name);
444                 return -E2BIG;
445         }
446         if (!priv->caps.queue_counters_valid) {
447                 DRV_LOG(ERR, "Virtq statistics is not supported for device %s.",
448                         vdev->device->name);
449                 return -ENOTSUP;
450         }
451         return mlx5_vdpa_virtq_stats_reset(priv, qid);
452 }
453
454 static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {
455         .get_queue_num = mlx5_vdpa_get_queue_num,
456         .get_features = mlx5_vdpa_get_vdpa_features,
457         .get_protocol_features = mlx5_vdpa_get_protocol_features,
458         .dev_conf = mlx5_vdpa_dev_config,
459         .dev_close = mlx5_vdpa_dev_close,
460         .set_vring_state = mlx5_vdpa_set_vring_state,
461         .set_features = mlx5_vdpa_features_set,
462         .migration_done = NULL,
463         .get_vfio_group_fd = NULL,
464         .get_vfio_device_fd = mlx5_vdpa_get_device_fd,
465         .get_notify_area = mlx5_vdpa_get_notify_area,
466         .get_stats_names = mlx5_vdpa_get_stats_names,
467         .get_stats = mlx5_vdpa_get_stats,
468         .reset_stats = mlx5_vdpa_reset_stats,
469 };
470
471 static struct ibv_device *
472 mlx5_vdpa_get_ib_device_match(struct rte_pci_addr *addr)
473 {
474         int n;
475         struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
476         struct ibv_device *ibv_match = NULL;
477
478         if (!ibv_list) {
479                 rte_errno = ENOSYS;
480                 return NULL;
481         }
482         while (n-- > 0) {
483                 struct rte_pci_addr pci_addr;
484
485                 DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
486                 if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &pci_addr))
487                         continue;
488                 if (rte_pci_addr_cmp(addr, &pci_addr))
489                         continue;
490                 ibv_match = ibv_list[n];
491                 break;
492         }
493         if (!ibv_match)
494                 rte_errno = ENOENT;
495         mlx5_glue->free_device_list(ibv_list);
496         return ibv_match;
497 }
498
499 /* Try to disable ROCE by Netlink\Devlink. */
500 static int
501 mlx5_vdpa_nl_roce_disable(const char *addr)
502 {
503         int nlsk_fd = mlx5_nl_init(NETLINK_GENERIC);
504         int devlink_id;
505         int enable;
506         int ret;
507
508         if (nlsk_fd < 0)
509                 return nlsk_fd;
510         devlink_id = mlx5_nl_devlink_family_id_get(nlsk_fd);
511         if (devlink_id < 0) {
512                 ret = devlink_id;
513                 DRV_LOG(DEBUG, "Failed to get devlink id for ROCE operations by"
514                         " Netlink.");
515                 goto close;
516         }
517         ret = mlx5_nl_enable_roce_get(nlsk_fd, devlink_id, addr, &enable);
518         if (ret) {
519                 DRV_LOG(DEBUG, "Failed to get ROCE enable by Netlink: %d.",
520                         ret);
521                 goto close;
522         } else if (!enable) {
523                 DRV_LOG(INFO, "ROCE has already disabled(Netlink).");
524                 goto close;
525         }
526         ret = mlx5_nl_enable_roce_set(nlsk_fd, devlink_id, addr, 0);
527         if (ret)
528                 DRV_LOG(DEBUG, "Failed to disable ROCE by Netlink: %d.", ret);
529         else
530                 DRV_LOG(INFO, "ROCE is disabled by Netlink successfully.");
531 close:
532         close(nlsk_fd);
533         return ret;
534 }
535
536 /* Try to disable ROCE by sysfs. */
537 static int
538 mlx5_vdpa_sys_roce_disable(const char *addr)
539 {
540         FILE *file_o;
541         int enable;
542         int ret;
543
544         MKSTR(file_p, "/sys/bus/pci/devices/%s/roce_enable", addr);
545         file_o = fopen(file_p, "rb");
546         if (!file_o) {
547                 rte_errno = ENOTSUP;
548                 return -ENOTSUP;
549         }
550         ret = fscanf(file_o, "%d", &enable);
551         if (ret != 1) {
552                 rte_errno = EINVAL;
553                 ret = EINVAL;
554                 goto close;
555         } else if (!enable) {
556                 ret = 0;
557                 DRV_LOG(INFO, "ROCE has already disabled(sysfs).");
558                 goto close;
559         }
560         fclose(file_o);
561         file_o = fopen(file_p, "wb");
562         if (!file_o) {
563                 rte_errno = ENOTSUP;
564                 return -ENOTSUP;
565         }
566         fprintf(file_o, "0\n");
567         ret = 0;
568 close:
569         if (ret)
570                 DRV_LOG(DEBUG, "Failed to disable ROCE by sysfs: %d.", ret);
571         else
572                 DRV_LOG(INFO, "ROCE is disabled by sysfs successfully.");
573         fclose(file_o);
574         return ret;
575 }
576
577 static int
578 mlx5_vdpa_roce_disable(struct rte_pci_addr *addr, struct ibv_device **ibv)
579 {
580         char addr_name[64] = {0};
581
582         rte_pci_device_name(addr, addr_name, sizeof(addr_name));
583         /* Firstly try to disable ROCE by Netlink and fallback to sysfs. */
584         if (mlx5_vdpa_nl_roce_disable(addr_name) == 0 ||
585             mlx5_vdpa_sys_roce_disable(addr_name) == 0) {
586                 /*
587                  * Succeed to disable ROCE, wait for the IB device to appear
588                  * again after reload.
589                  */
590                 int r;
591                 struct ibv_device *ibv_new;
592
593                 for (r = MLX5_VDPA_MAX_RETRIES; r; r--) {
594                         ibv_new = mlx5_vdpa_get_ib_device_match(addr);
595                         if (ibv_new) {
596                                 *ibv = ibv_new;
597                                 return 0;
598                         }
599                         usleep(MLX5_VDPA_USEC);
600                 }
601                 DRV_LOG(ERR, "Cannot much device %s after ROCE disable, "
602                         "retries exceed %d", addr_name, MLX5_VDPA_MAX_RETRIES);
603                 rte_errno = EAGAIN;
604         }
605         return -rte_errno;
606 }
607
608 static int
609 mlx5_vdpa_args_check_handler(const char *key, const char *val, void *opaque)
610 {
611         struct mlx5_vdpa_priv *priv = opaque;
612         unsigned long tmp;
613
614         if (strcmp(key, "class") == 0)
615                 return 0;
616         errno = 0;
617         tmp = strtoul(val, NULL, 0);
618         if (errno) {
619                 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
620                 return -errno;
621         }
622         if (strcmp(key, "event_mode") == 0) {
623                 if (tmp <= MLX5_VDPA_EVENT_MODE_ONLY_INTERRUPT)
624                         priv->event_mode = (int)tmp;
625                 else
626                         DRV_LOG(WARNING, "Invalid event_mode %s.", val);
627         } else if (strcmp(key, "event_us") == 0) {
628                 priv->event_us = (uint32_t)tmp;
629         } else if (strcmp(key, "no_traffic_time") == 0) {
630                 priv->no_traffic_time_s = (uint32_t)tmp;
631         } else {
632                 DRV_LOG(WARNING, "Invalid key %s.", key);
633         }
634         return 0;
635 }
636
637 static void
638 mlx5_vdpa_config_get(struct rte_devargs *devargs, struct mlx5_vdpa_priv *priv)
639 {
640         struct rte_kvargs *kvlist;
641
642         priv->event_mode = MLX5_VDPA_EVENT_MODE_DYNAMIC_TIMER;
643         priv->event_us = 0;
644         priv->no_traffic_time_s = MLX5_VDPA_DEFAULT_NO_TRAFFIC_TIME_S;
645         if (devargs == NULL)
646                 return;
647         kvlist = rte_kvargs_parse(devargs->args, NULL);
648         if (kvlist == NULL)
649                 return;
650         rte_kvargs_process(kvlist, NULL, mlx5_vdpa_args_check_handler, priv);
651         rte_kvargs_free(kvlist);
652         if (!priv->event_us) {
653                 if (priv->event_mode == MLX5_VDPA_EVENT_MODE_DYNAMIC_TIMER)
654                         priv->event_us = MLX5_VDPA_DEFAULT_TIMER_STEP_US;
655                 else if (priv->event_mode == MLX5_VDPA_EVENT_MODE_FIXED_TIMER)
656                         priv->event_us = MLX5_VDPA_DEFAULT_TIMER_DELAY_US;
657         }
658         DRV_LOG(DEBUG, "event mode is %d.", priv->event_mode);
659         DRV_LOG(DEBUG, "event_us is %u us.", priv->event_us);
660         DRV_LOG(DEBUG, "no traffic time is %u s.", priv->no_traffic_time_s);
661 }
662
663 /**
664  * DPDK callback to register a mlx5 PCI device.
665  *
666  * This function spawns vdpa device out of a given PCI device.
667  *
668  * @param[in] pci_drv
669  *   PCI driver structure (mlx5_vpda_driver).
670  * @param[in] pci_dev
671  *   PCI device information.
672  *
673  * @return
674  *   0 on success, 1 to skip this driver, a negative errno value otherwise
675  *   and rte_errno is set.
676  */
677 static int
678 mlx5_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
679                     struct rte_pci_device *pci_dev __rte_unused)
680 {
681         struct ibv_device *ibv;
682         struct mlx5_vdpa_priv *priv = NULL;
683         struct ibv_context *ctx = NULL;
684         struct mlx5_hca_attr attr;
685         int ret;
686
687         ibv = mlx5_vdpa_get_ib_device_match(&pci_dev->addr);
688         if (!ibv) {
689                 DRV_LOG(ERR, "No matching IB device for PCI slot "
690                         PCI_PRI_FMT ".", pci_dev->addr.domain,
691                         pci_dev->addr.bus, pci_dev->addr.devid,
692                         pci_dev->addr.function);
693                 return -rte_errno;
694         } else {
695                 DRV_LOG(INFO, "PCI information matches for device \"%s\".",
696                         ibv->name);
697         }
698         if (mlx5_vdpa_roce_disable(&pci_dev->addr, &ibv) != 0) {
699                 DRV_LOG(WARNING, "Failed to disable ROCE for \"%s\".",
700                         ibv->name);
701                 return -rte_errno;
702         }
703         ctx = mlx5_glue->dv_open_device(ibv);
704         if (!ctx) {
705                 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
706                 rte_errno = ENODEV;
707                 return -rte_errno;
708         }
709         ret = mlx5_devx_cmd_query_hca_attr(ctx, &attr);
710         if (ret) {
711                 DRV_LOG(ERR, "Unable to read HCA capabilities.");
712                 rte_errno = ENOTSUP;
713                 goto error;
714         } else if (!attr.vdpa.valid || !attr.vdpa.max_num_virtio_queues) {
715                 DRV_LOG(ERR, "Not enough capabilities to support vdpa, maybe "
716                         "old FW/OFED version?");
717                 rte_errno = ENOTSUP;
718                 goto error;
719         }
720         if (!attr.vdpa.queue_counters_valid)
721                 DRV_LOG(DEBUG, "No capability to support virtq statistics.");
722         priv = rte_zmalloc("mlx5 vDPA device private", sizeof(*priv) +
723                            sizeof(struct mlx5_vdpa_virtq) *
724                            attr.vdpa.max_num_virtio_queues * 2,
725                            RTE_CACHE_LINE_SIZE);
726         if (!priv) {
727                 DRV_LOG(ERR, "Failed to allocate private memory.");
728                 rte_errno = ENOMEM;
729                 goto error;
730         }
731         priv->caps = attr.vdpa;
732         priv->log_max_rqt_size = attr.log_max_rqt_size;
733         priv->ctx = ctx;
734         priv->pci_dev = pci_dev;
735         priv->var = mlx5_glue->dv_alloc_var(ctx, 0);
736         if (!priv->var) {
737                 DRV_LOG(ERR, "Failed to allocate VAR %u.\n", errno);
738                 goto error;
739         }
740         priv->vdev = rte_vdpa_register_device(&pci_dev->device,
741                         &mlx5_vdpa_ops);
742         if (priv->vdev == NULL) {
743                 DRV_LOG(ERR, "Failed to register vDPA device.");
744                 rte_errno = rte_errno ? rte_errno : EINVAL;
745                 goto error;
746         }
747         mlx5_vdpa_config_get(pci_dev->device.devargs, priv);
748         SLIST_INIT(&priv->mr_list);
749         pthread_mutex_init(&priv->vq_config_lock, NULL);
750         pthread_mutex_lock(&priv_list_lock);
751         TAILQ_INSERT_TAIL(&priv_list, priv, next);
752         pthread_mutex_unlock(&priv_list_lock);
753         return 0;
754
755 error:
756         if (priv) {
757                 if (priv->var)
758                         mlx5_glue->dv_free_var(priv->var);
759                 rte_free(priv);
760         }
761         if (ctx)
762                 mlx5_glue->close_device(ctx);
763         return -rte_errno;
764 }
765
766 /**
767  * DPDK callback to remove a PCI device.
768  *
769  * This function removes all vDPA devices belong to a given PCI device.
770  *
771  * @param[in] pci_dev
772  *   Pointer to the PCI device.
773  *
774  * @return
775  *   0 on success, the function cannot fail.
776  */
777 static int
778 mlx5_vdpa_pci_remove(struct rte_pci_device *pci_dev)
779 {
780         struct mlx5_vdpa_priv *priv = NULL;
781         int found = 0;
782
783         pthread_mutex_lock(&priv_list_lock);
784         TAILQ_FOREACH(priv, &priv_list, next) {
785                 if (!rte_pci_addr_cmp(&priv->pci_dev->addr, &pci_dev->addr)) {
786                         found = 1;
787                         break;
788                 }
789         }
790         if (found)
791                 TAILQ_REMOVE(&priv_list, priv, next);
792         pthread_mutex_unlock(&priv_list_lock);
793         if (found) {
794                 if (priv->configured)
795                         mlx5_vdpa_dev_close(priv->vid);
796                 if (priv->var) {
797                         mlx5_glue->dv_free_var(priv->var);
798                         priv->var = NULL;
799                 }
800                 mlx5_glue->close_device(priv->ctx);
801                 pthread_mutex_destroy(&priv->vq_config_lock);
802                 rte_free(priv);
803         }
804         return 0;
805 }
806
807 static const struct rte_pci_id mlx5_vdpa_pci_id_map[] = {
808         {
809                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
810                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6)
811         },
812         {
813                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
814                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF)
815         },
816         {
817                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
818                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
819         },
820         {
821                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
822                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF)
823         },
824         {
825                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
826                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
827         },
828         {
829                 .vendor_id = 0
830         }
831 };
832
833 static struct mlx5_pci_driver mlx5_vdpa_driver = {
834         .driver_class = MLX5_CLASS_VDPA,
835         .pci_driver = {
836                 .driver = {
837                         .name = "mlx5_vdpa",
838                 },
839                 .id_table = mlx5_vdpa_pci_id_map,
840                 .probe = mlx5_vdpa_pci_probe,
841                 .remove = mlx5_vdpa_pci_remove,
842                 .drv_flags = 0,
843         },
844 };
845
846 RTE_LOG_REGISTER(mlx5_vdpa_logtype, pmd.vdpa.mlx5, NOTICE)
847
848 /**
849  * Driver initialization routine.
850  */
851 RTE_INIT(rte_mlx5_vdpa_init)
852 {
853         mlx5_common_init();
854         if (mlx5_glue)
855                 mlx5_pci_driver_register(&mlx5_vdpa_driver);
856 }
857
858 RTE_PMD_EXPORT_NAME(net_mlx5_vdpa, __COUNTER__);
859 RTE_PMD_REGISTER_PCI_TABLE(net_mlx5_vdpa, mlx5_vdpa_pci_id_map);
860 RTE_PMD_REGISTER_KMOD_DEP(net_mlx5_vdpa, "* ib_uverbs & mlx5_core & mlx5_ib");