1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020-2021 Xilinx, Inc.
8 #include <rte_malloc.h>
11 #include <rte_vhost.h>
13 #include <vdpa_driver.h>
16 #include "sfc_vdpa_ops.h"
19 /* These protocol features are needed to enable notifier ctrl */
20 #define SFC_VDPA_PROTOCOL_FEATURES \
21 ((1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK) | \
22 (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
23 (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
24 (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
25 (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD))
28 * Set of features which are enabled by default.
29 * Protocol feature bit is needed to enable notification notifier ctrl.
31 #define SFC_VDPA_DEFAULT_FEATURES \
32 (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)
34 #define SFC_VDPA_MSIX_IRQ_SET_BUF_LEN \
35 (sizeof(struct vfio_irq_set) + \
36 sizeof(int) * (SFC_VDPA_MAX_QUEUE_PAIRS * 2 + 1))
38 /* It will be used for target VF when calling function is not PF */
39 #define SFC_VDPA_VF_NULL 0xFFFF
42 sfc_vdpa_get_device_features(struct sfc_vdpa_ops_data *ops_data)
45 uint64_t dev_features;
48 nic = sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle)->nic;
50 rc = efx_virtio_get_features(nic, EFX_VIRTIO_DEVICE_TYPE_NET,
53 sfc_vdpa_err(ops_data->dev_handle,
54 "could not read device feature: %s",
59 ops_data->dev_features = dev_features;
61 sfc_vdpa_info(ops_data->dev_handle,
62 "device supported virtio features : 0x%" PRIx64,
63 ops_data->dev_features);
69 hva_to_gpa(int vid, uint64_t hva)
71 struct rte_vhost_memory *vhost_mem = NULL;
72 struct rte_vhost_mem_region *mem_reg = NULL;
76 if (rte_vhost_get_mem_table(vid, &vhost_mem) < 0)
79 for (i = 0; i < vhost_mem->nregions; i++) {
80 mem_reg = &vhost_mem->regions[i];
82 if (hva >= mem_reg->host_user_addr &&
83 hva < mem_reg->host_user_addr + mem_reg->size) {
84 gpa = (hva - mem_reg->host_user_addr) +
85 mem_reg->guest_phys_addr;
96 sfc_vdpa_enable_vfio_intr(struct sfc_vdpa_ops_data *ops_data)
101 uint32_t i, num_vring;
102 struct rte_vhost_vring vring;
103 struct vfio_irq_set *irq_set;
104 struct rte_pci_device *pci_dev;
105 char irq_set_buf[SFC_VDPA_MSIX_IRQ_SET_BUF_LEN];
108 num_vring = rte_vhost_get_vring_num(ops_data->vid);
109 dev = ops_data->dev_handle;
110 vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
111 pci_dev = sfc_vdpa_adapter_by_dev_handle(dev)->pdev;
113 irq_set = (struct vfio_irq_set *)irq_set_buf;
114 irq_set->argsz = sizeof(irq_set_buf);
115 irq_set->count = num_vring + 1;
116 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
117 VFIO_IRQ_SET_ACTION_TRIGGER;
118 irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
120 irq_fd_ptr = (int *)&irq_set->data;
121 irq_fd_ptr[RTE_INTR_VEC_ZERO_OFFSET] =
122 rte_intr_fd_get(pci_dev->intr_handle);
124 for (i = 0; i < num_vring; i++) {
125 rc = rte_vhost_get_vhost_vring(ops_data->vid, i, &vring);
129 irq_fd_ptr[RTE_INTR_VEC_RXTX_OFFSET + i] = vring.callfd;
132 rc = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
134 sfc_vdpa_err(ops_data->dev_handle,
135 "error enabling MSI-X interrupts: %s",
144 sfc_vdpa_disable_vfio_intr(struct sfc_vdpa_ops_data *ops_data)
148 struct vfio_irq_set irq_set;
151 dev = ops_data->dev_handle;
152 vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
154 irq_set.argsz = sizeof(irq_set);
156 irq_set.flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
157 irq_set.index = VFIO_PCI_MSIX_IRQ_INDEX;
160 rc = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, &irq_set);
162 sfc_vdpa_err(ops_data->dev_handle,
163 "error disabling MSI-X interrupts: %s",
172 sfc_vdpa_get_vring_info(struct sfc_vdpa_ops_data *ops_data,
173 int vq_num, struct sfc_vdpa_vring_info *vring)
177 struct rte_vhost_vring vq;
179 rc = rte_vhost_get_vhost_vring(ops_data->vid, vq_num, &vq);
181 sfc_vdpa_err(ops_data->dev_handle,
182 "get vhost vring failed: %s", rte_strerror(rc));
186 gpa = hva_to_gpa(ops_data->vid, (uint64_t)(uintptr_t)vq.desc);
188 sfc_vdpa_err(ops_data->dev_handle,
189 "fail to get GPA for descriptor ring.");
194 gpa = hva_to_gpa(ops_data->vid, (uint64_t)(uintptr_t)vq.avail);
196 sfc_vdpa_err(ops_data->dev_handle,
197 "fail to get GPA for available ring.");
202 gpa = hva_to_gpa(ops_data->vid, (uint64_t)(uintptr_t)vq.used);
204 sfc_vdpa_err(ops_data->dev_handle,
205 "fail to get GPA for used ring.");
210 vring->size = vq.size;
212 rc = rte_vhost_get_vring_base(ops_data->vid, vq_num,
213 &vring->last_avail_idx,
214 &vring->last_used_idx);
220 sfc_vdpa_virtq_start(struct sfc_vdpa_ops_data *ops_data, int vq_num)
224 struct sfc_vdpa_vring_info vring;
225 efx_virtio_vq_cfg_t vq_cfg;
226 efx_virtio_vq_dyncfg_t vq_dyncfg;
228 vq = ops_data->vq_cxt[vq_num].vq;
232 rc = sfc_vdpa_get_vring_info(ops_data, vq_num, &vring);
234 sfc_vdpa_err(ops_data->dev_handle,
235 "get vring info failed: %s", rte_strerror(rc));
236 goto fail_vring_info;
239 vq_cfg.evvc_target_vf = SFC_VDPA_VF_NULL;
241 /* even virtqueue for RX and odd for TX */
243 vq_cfg.evvc_type = EFX_VIRTIO_VQ_TYPE_NET_TXQ;
244 sfc_vdpa_info(ops_data->dev_handle,
245 "configure virtqueue # %d (TXQ)", vq_num);
247 vq_cfg.evvc_type = EFX_VIRTIO_VQ_TYPE_NET_RXQ;
248 sfc_vdpa_info(ops_data->dev_handle,
249 "configure virtqueue # %d (RXQ)", vq_num);
252 vq_cfg.evvc_vq_num = vq_num;
253 vq_cfg.evvc_desc_tbl_addr = vring.desc;
254 vq_cfg.evvc_avail_ring_addr = vring.avail;
255 vq_cfg.evvc_used_ring_addr = vring.used;
256 vq_cfg.evvc_vq_size = vring.size;
258 vq_dyncfg.evvd_vq_pidx = vring.last_used_idx;
259 vq_dyncfg.evvd_vq_cidx = vring.last_avail_idx;
261 /* MSI-X vector is function-relative */
262 vq_cfg.evvc_msix_vector = RTE_INTR_VEC_RXTX_OFFSET + vq_num;
263 if (ops_data->vdpa_context == SFC_VDPA_AS_VF)
264 vq_cfg.evvc_pas_id = 0;
265 vq_cfg.evcc_features = ops_data->dev_features &
266 ops_data->req_features;
268 /* Start virtqueue */
269 rc = efx_virtio_qstart(vq, &vq_cfg, &vq_dyncfg);
271 /* destroy virtqueue */
272 sfc_vdpa_err(ops_data->dev_handle,
273 "virtqueue start failed: %s",
275 efx_virtio_qdestroy(vq);
276 goto fail_virtio_qstart;
279 sfc_vdpa_info(ops_data->dev_handle,
280 "virtqueue started successfully for vq_num %d", vq_num);
282 ops_data->vq_cxt[vq_num].enable = B_TRUE;
292 sfc_vdpa_virtq_stop(struct sfc_vdpa_ops_data *ops_data, int vq_num)
295 efx_virtio_vq_dyncfg_t vq_idx;
298 if (ops_data->vq_cxt[vq_num].enable != B_TRUE)
301 vq = ops_data->vq_cxt[vq_num].vq;
306 rc = efx_virtio_qstop(vq, &vq_idx);
308 ops_data->vq_cxt[vq_num].cidx = vq_idx.evvd_vq_cidx;
309 ops_data->vq_cxt[vq_num].pidx = vq_idx.evvd_vq_pidx;
311 ops_data->vq_cxt[vq_num].enable = B_FALSE;
317 sfc_vdpa_configure(struct sfc_vdpa_ops_data *ops_data)
326 dev = ops_data->dev_handle;
327 nic = sfc_vdpa_adapter_by_dev_handle(dev)->nic;
329 SFC_EFX_ASSERT(ops_data->state == SFC_VDPA_STATE_INITIALIZED);
331 ops_data->state = SFC_VDPA_STATE_CONFIGURING;
333 nr_vring = rte_vhost_get_vring_num(ops_data->vid);
335 (sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count * 2);
337 /* number of vring should not be more than supported max vq count */
338 if (nr_vring > max_vring_cnt) {
340 "nr_vring (%d) is > max vring count (%d)",
341 nr_vring, max_vring_cnt);
345 rc = sfc_vdpa_dma_map(ops_data, true);
348 "DMA map failed: %s", rte_strerror(rc));
352 for (i = 0; i < nr_vring; i++) {
353 rc = efx_virtio_qcreate(nic, &vq);
354 if ((rc != 0) || (vq == NULL)) {
356 "virtqueue create failed: %s",
361 /* store created virtqueue context */
362 ops_data->vq_cxt[i].vq = vq;
365 ops_data->vq_count = i;
367 ops_data->state = SFC_VDPA_STATE_CONFIGURED;
372 sfc_vdpa_dma_map(ops_data, false);
376 ops_data->state = SFC_VDPA_STATE_INITIALIZED;
382 sfc_vdpa_close(struct sfc_vdpa_ops_data *ops_data)
386 if (ops_data->state != SFC_VDPA_STATE_CONFIGURED)
389 ops_data->state = SFC_VDPA_STATE_CLOSING;
391 for (i = 0; i < ops_data->vq_count; i++) {
392 if (ops_data->vq_cxt[i].vq == NULL)
395 efx_virtio_qdestroy(ops_data->vq_cxt[i].vq);
398 sfc_vdpa_dma_map(ops_data, false);
400 ops_data->state = SFC_VDPA_STATE_INITIALIZED;
404 sfc_vdpa_stop(struct sfc_vdpa_ops_data *ops_data)
409 if (ops_data->state != SFC_VDPA_STATE_STARTED)
412 ops_data->state = SFC_VDPA_STATE_STOPPING;
414 for (i = 0; i < ops_data->vq_count; i++) {
415 rc = sfc_vdpa_virtq_stop(ops_data, i);
420 sfc_vdpa_disable_vfio_intr(ops_data);
422 ops_data->state = SFC_VDPA_STATE_CONFIGURED;
426 sfc_vdpa_start(struct sfc_vdpa_ops_data *ops_data)
431 SFC_EFX_ASSERT(ops_data->state == SFC_VDPA_STATE_CONFIGURED);
433 sfc_vdpa_log_init(ops_data->dev_handle, "entry");
435 ops_data->state = SFC_VDPA_STATE_STARTING;
437 sfc_vdpa_log_init(ops_data->dev_handle, "enable interrupts");
438 rc = sfc_vdpa_enable_vfio_intr(ops_data);
440 sfc_vdpa_err(ops_data->dev_handle,
441 "vfio intr allocation failed: %s",
443 goto fail_enable_vfio_intr;
446 rte_vhost_get_negotiated_features(ops_data->vid,
447 &ops_data->req_features);
449 sfc_vdpa_info(ops_data->dev_handle,
450 "negotiated feature : 0x%" PRIx64,
451 ops_data->req_features);
453 for (i = 0; i < ops_data->vq_count; i++) {
454 sfc_vdpa_log_init(ops_data->dev_handle,
455 "starting vq# %d", i);
456 rc = sfc_vdpa_virtq_start(ops_data, i);
461 ops_data->state = SFC_VDPA_STATE_STARTED;
463 sfc_vdpa_log_init(ops_data->dev_handle, "done");
468 /* stop already started virtqueues */
469 for (j = 0; j < i; j++)
470 sfc_vdpa_virtq_stop(ops_data, j);
471 sfc_vdpa_disable_vfio_intr(ops_data);
473 fail_enable_vfio_intr:
474 ops_data->state = SFC_VDPA_STATE_CONFIGURED;
480 sfc_vdpa_get_queue_num(struct rte_vdpa_device *vdpa_dev, uint32_t *queue_num)
482 struct sfc_vdpa_ops_data *ops_data;
485 ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
486 if (ops_data == NULL)
489 dev = ops_data->dev_handle;
490 *queue_num = sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count;
492 sfc_vdpa_info(dev, "vDPA ops get_queue_num :: supported queue num : %u",
499 sfc_vdpa_get_features(struct rte_vdpa_device *vdpa_dev, uint64_t *features)
501 struct sfc_vdpa_ops_data *ops_data;
503 ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
504 if (ops_data == NULL)
507 *features = ops_data->drv_features;
509 sfc_vdpa_info(ops_data->dev_handle,
510 "vDPA ops get_feature :: features : 0x%" PRIx64,
517 sfc_vdpa_get_protocol_features(struct rte_vdpa_device *vdpa_dev,
520 struct sfc_vdpa_ops_data *ops_data;
522 ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
523 if (ops_data == NULL)
526 *features = SFC_VDPA_PROTOCOL_FEATURES;
528 sfc_vdpa_info(ops_data->dev_handle,
529 "vDPA ops get_protocol_feature :: features : 0x%" PRIx64,
536 sfc_vdpa_dev_config(int vid)
538 struct rte_vdpa_device *vdpa_dev;
540 struct sfc_vdpa_ops_data *ops_data;
542 vdpa_dev = rte_vhost_get_vdpa_device(vid);
544 ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
545 if (ops_data == NULL) {
546 sfc_vdpa_err(ops_data->dev_handle,
547 "invalid vDPA device : %p, vid : %d",
552 sfc_vdpa_log_init(ops_data->dev_handle, "entry");
556 sfc_vdpa_adapter_lock(ops_data->dev_handle);
558 sfc_vdpa_log_init(ops_data->dev_handle, "configuring");
559 rc = sfc_vdpa_configure(ops_data);
561 goto fail_vdpa_config;
563 sfc_vdpa_log_init(ops_data->dev_handle, "starting");
564 rc = sfc_vdpa_start(ops_data);
566 goto fail_vdpa_start;
568 sfc_vdpa_adapter_unlock(ops_data->dev_handle);
570 sfc_vdpa_log_init(ops_data->dev_handle, "vhost notifier ctrl");
571 if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
572 sfc_vdpa_info(ops_data->dev_handle,
573 "vDPA (%s): software relay for notify is used.",
574 vdpa_dev->device->name);
576 sfc_vdpa_log_init(ops_data->dev_handle, "done");
581 sfc_vdpa_close(ops_data);
584 sfc_vdpa_adapter_unlock(ops_data->dev_handle);
590 sfc_vdpa_dev_close(int vid)
592 struct rte_vdpa_device *vdpa_dev;
593 struct sfc_vdpa_ops_data *ops_data;
595 vdpa_dev = rte_vhost_get_vdpa_device(vid);
597 ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
598 if (ops_data == NULL) {
599 sfc_vdpa_err(ops_data->dev_handle,
600 "invalid vDPA device : %p, vid : %d",
605 sfc_vdpa_adapter_lock(ops_data->dev_handle);
607 sfc_vdpa_stop(ops_data);
608 sfc_vdpa_close(ops_data);
610 sfc_vdpa_adapter_unlock(ops_data->dev_handle);
616 sfc_vdpa_set_vring_state(int vid, int vring, int state)
626 sfc_vdpa_set_features(int vid)
634 sfc_vdpa_get_vfio_device_fd(int vid)
636 struct rte_vdpa_device *vdpa_dev;
637 struct sfc_vdpa_ops_data *ops_data;
641 vdpa_dev = rte_vhost_get_vdpa_device(vid);
643 ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
644 if (ops_data == NULL)
647 dev = ops_data->dev_handle;
648 vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
650 sfc_vdpa_info(dev, "vDPA ops get_vfio_device_fd :: vfio fd : %d",
656 static struct rte_vdpa_dev_ops sfc_vdpa_ops = {
657 .get_queue_num = sfc_vdpa_get_queue_num,
658 .get_features = sfc_vdpa_get_features,
659 .get_protocol_features = sfc_vdpa_get_protocol_features,
660 .dev_conf = sfc_vdpa_dev_config,
661 .dev_close = sfc_vdpa_dev_close,
662 .set_vring_state = sfc_vdpa_set_vring_state,
663 .set_features = sfc_vdpa_set_features,
664 .get_vfio_device_fd = sfc_vdpa_get_vfio_device_fd,
667 struct sfc_vdpa_ops_data *
668 sfc_vdpa_device_init(void *dev_handle, enum sfc_vdpa_context context)
670 struct sfc_vdpa_ops_data *ops_data;
671 struct rte_pci_device *pci_dev;
674 /* Create vDPA ops context */
675 ops_data = rte_zmalloc("vdpa", sizeof(struct sfc_vdpa_ops_data), 0);
676 if (ops_data == NULL)
679 ops_data->vdpa_context = context;
680 ops_data->dev_handle = dev_handle;
682 pci_dev = sfc_vdpa_adapter_by_dev_handle(dev_handle)->pdev;
684 /* Register vDPA Device */
685 sfc_vdpa_log_init(dev_handle, "register vDPA device");
687 rte_vdpa_register_device(&pci_dev->device, &sfc_vdpa_ops);
688 if (ops_data->vdpa_dev == NULL) {
689 sfc_vdpa_err(dev_handle, "vDPA device registration failed");
690 goto fail_register_device;
693 /* Read supported device features */
694 sfc_vdpa_log_init(dev_handle, "get device feature");
695 rc = sfc_vdpa_get_device_features(ops_data);
697 goto fail_get_dev_feature;
699 /* Driver features are superset of device supported feature
700 * and any additional features supported by the driver.
702 ops_data->drv_features =
703 ops_data->dev_features | SFC_VDPA_DEFAULT_FEATURES;
705 ops_data->state = SFC_VDPA_STATE_INITIALIZED;
709 fail_get_dev_feature:
710 rte_vdpa_unregister_device(ops_data->vdpa_dev);
712 fail_register_device:
718 sfc_vdpa_device_fini(struct sfc_vdpa_ops_data *ops_data)
720 rte_vdpa_unregister_device(ops_data->vdpa_dev);