1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2020 Intel Corporation
16 #include <rte_debug.h>
19 #include <rte_devargs.h>
22 #include <rte_errno.h>
23 #include <rte_kvargs.h>
24 #include <rte_malloc.h>
25 #include <rte_cycles.h>
28 #include <rte_bus_pci.h>
29 #include <rte_eventdev.h>
30 #include <eventdev_pmd.h>
31 #include <eventdev_pmd_pci.h>
32 #include <rte_memory.h>
33 #include <rte_string_fns.h>
35 #define DLB2_USE_NEW_HEADERS /* TEMPORARY FOR MERGE */
37 #include "../dlb2_priv.h"
38 #include "../dlb2_iface.h"
39 #include "../dlb2_inline_fns.h"
40 #include "dlb2_main.h"
41 #include "base/dlb2_hw_types_new.h"
42 #include "base/dlb2_osdep.h"
43 #include "base/dlb2_resource.h"
45 static const char *event_dlb2_pf_name = RTE_STR(EVDEV_DLB2_NAME_PMD);
48 dlb2_pf_low_level_io_init(void)
51 /* Addresses will be initialized at port create */
52 for (i = 0; i < DLB2_MAX_NUM_PORTS(DLB2_HW_V2_5); i++) {
53 /* First directed ports */
54 dlb2_port[i][DLB2_DIR_PORT].pp_addr = NULL;
55 dlb2_port[i][DLB2_DIR_PORT].cq_base = NULL;
56 dlb2_port[i][DLB2_DIR_PORT].mmaped = true;
58 /* Now load balanced ports */
59 dlb2_port[i][DLB2_LDB_PORT].pp_addr = NULL;
60 dlb2_port[i][DLB2_LDB_PORT].cq_base = NULL;
61 dlb2_port[i][DLB2_LDB_PORT].mmaped = true;
66 dlb2_pf_open(struct dlb2_hw_dev *handle, const char *name)
75 dlb2_pf_get_device_version(struct dlb2_hw_dev *handle,
78 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
80 *revision = dlb2_dev->revision;
86 dlb2_pf_hardware_init(struct dlb2_hw_dev *handle)
88 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
90 dlb2_hw_enable_sparse_ldb_cq_mode(&dlb2_dev->hw);
91 dlb2_hw_enable_sparse_dir_cq_mode(&dlb2_dev->hw);
95 dlb2_pf_get_num_resources(struct dlb2_hw_dev *handle,
96 struct dlb2_get_num_resources_args *rsrcs)
98 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
100 return dlb2_hw_get_num_resources(&dlb2_dev->hw, rsrcs, false, 0);
104 dlb2_pf_get_cq_poll_mode(struct dlb2_hw_dev *handle,
105 enum dlb2_cq_poll_modes *mode)
107 RTE_SET_USED(handle);
109 *mode = DLB2_CQ_POLL_MODE_SPARSE;
115 dlb2_pf_sched_domain_create(struct dlb2_hw_dev *handle,
116 struct dlb2_create_sched_domain_args *arg)
118 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
119 struct dlb2_cmd_response response = {0};
122 DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
124 if (dlb2_dev->domain_reset_failed) {
125 response.status = DLB2_ST_DOMAIN_RESET_FAILED;
130 ret = dlb2_pf_create_sched_domain(&dlb2_dev->hw, arg, &response);
136 arg->response = response;
138 DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
145 dlb2_pf_domain_reset(struct dlb2_eventdev *dlb2)
147 struct dlb2_dev *dlb2_dev;
150 dlb2_dev = (struct dlb2_dev *)dlb2->qm_instance.pf_dev;
151 ret = dlb2_pf_reset_domain(&dlb2_dev->hw, dlb2->qm_instance.domain_id);
153 DLB2_LOG_ERR("dlb2_pf_reset_domain err %d", ret);
157 dlb2_pf_ldb_queue_create(struct dlb2_hw_dev *handle,
158 struct dlb2_create_ldb_queue_args *cfg)
160 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
161 struct dlb2_cmd_response response = {0};
164 DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
166 ret = dlb2_pf_create_ldb_queue(&dlb2_dev->hw,
171 cfg->response = response;
173 DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
180 dlb2_pf_get_sn_occupancy(struct dlb2_hw_dev *handle,
181 struct dlb2_get_sn_occupancy_args *args)
183 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
184 struct dlb2_cmd_response response = {0};
187 ret = dlb2_get_group_sequence_number_occupancy(&dlb2_dev->hw,
193 args->response = response;
199 dlb2_pf_get_sn_allocation(struct dlb2_hw_dev *handle,
200 struct dlb2_get_sn_allocation_args *args)
202 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
203 struct dlb2_cmd_response response = {0};
206 ret = dlb2_get_group_sequence_numbers(&dlb2_dev->hw, args->group);
211 args->response = response;
217 dlb2_pf_set_sn_allocation(struct dlb2_hw_dev *handle,
218 struct dlb2_set_sn_allocation_args *args)
220 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
221 struct dlb2_cmd_response response = {0};
224 ret = dlb2_set_group_sequence_numbers(&dlb2_dev->hw, args->group,
229 args->response = response;
235 dlb2_alloc_coherent_aligned(const struct rte_memzone **mz, uintptr_t *phys,
236 size_t size, int align)
238 char mz_name[RTE_MEMZONE_NAMESIZE];
239 uint32_t core_id = rte_lcore_id();
240 unsigned int socket_id;
242 snprintf(mz_name, sizeof(mz_name) - 1, "event_dlb2_pf_%lx",
243 (unsigned long)rte_get_timer_cycles());
244 if (core_id == (unsigned int)LCORE_ID_ANY)
245 core_id = rte_get_main_lcore();
246 socket_id = rte_lcore_to_socket_id(core_id);
247 *mz = rte_memzone_reserve_aligned(mz_name, size, socket_id,
248 RTE_MEMZONE_IOVA_CONTIG, align);
250 DLB2_LOG_DBG("Unable to allocate DMA memory of size %zu bytes - %s\n",
251 size, rte_strerror(rte_errno));
260 dlb2_pf_ldb_port_create(struct dlb2_hw_dev *handle,
261 struct dlb2_create_ldb_port_args *cfg,
262 enum dlb2_cq_poll_modes poll_mode)
264 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
265 struct dlb2_cmd_response response = {0};
266 struct dlb2_port_memory port_memory;
267 int ret, cq_alloc_depth;
269 const struct rte_memzone *mz;
275 DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
277 if (poll_mode == DLB2_CQ_POLL_MODE_STD)
278 qe_sz = sizeof(struct dlb2_dequeue_qe);
280 qe_sz = RTE_CACHE_LINE_SIZE;
282 /* Calculate the port memory required, and round up to the nearest
285 cq_alloc_depth = RTE_MAX(cfg->cq_depth, DLB2_MIN_HARDWARE_CQ_DEPTH);
286 alloc_sz = cq_alloc_depth * qe_sz;
287 alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
289 port_base = dlb2_alloc_coherent_aligned(&mz, &cq_base, alloc_sz,
290 rte_mem_page_size());
291 if (port_base == NULL)
294 /* Lock the page in memory */
295 ret = rte_mem_lock_page(port_base);
297 DLB2_LOG_ERR("dlb2 pf pmd could not lock page for device i/o\n");
298 goto create_port_err;
301 memset(port_base, 0, alloc_sz);
303 ret = dlb2_pf_create_ldb_port(&dlb2_dev->hw,
309 goto create_port_err;
311 pp_base = (uintptr_t)dlb2_dev->hw.func_kva + PP_BASE(is_dir);
312 dlb2_port[response.id][DLB2_LDB_PORT].pp_addr =
313 (void *)(pp_base + (rte_mem_page_size() * response.id));
315 dlb2_port[response.id][DLB2_LDB_PORT].cq_base = (void *)(port_base);
316 memset(&port_memory, 0, sizeof(port_memory));
318 dlb2_port[response.id][DLB2_LDB_PORT].mz = mz;
320 dlb2_list_init_head(&port_memory.list);
322 cfg->response = response;
328 rte_memzone_free(mz);
330 DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
336 dlb2_pf_dir_port_create(struct dlb2_hw_dev *handle,
337 struct dlb2_create_dir_port_args *cfg,
338 enum dlb2_cq_poll_modes poll_mode)
340 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
341 struct dlb2_cmd_response response = {0};
342 struct dlb2_port_memory port_memory;
345 const struct rte_memzone *mz;
351 DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
353 if (poll_mode == DLB2_CQ_POLL_MODE_STD)
354 qe_sz = sizeof(struct dlb2_dequeue_qe);
356 qe_sz = RTE_CACHE_LINE_SIZE;
358 /* Calculate the port memory required, and round up to the nearest
361 alloc_sz = cfg->cq_depth * qe_sz;
362 alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
364 port_base = dlb2_alloc_coherent_aligned(&mz, &cq_base, alloc_sz,
365 rte_mem_page_size());
366 if (port_base == NULL)
369 /* Lock the page in memory */
370 ret = rte_mem_lock_page(port_base);
372 DLB2_LOG_ERR("dlb2 pf pmd could not lock page for device i/o\n");
373 goto create_port_err;
376 memset(port_base, 0, alloc_sz);
378 ret = dlb2_pf_create_dir_port(&dlb2_dev->hw,
384 goto create_port_err;
386 pp_base = (uintptr_t)dlb2_dev->hw.func_kva + PP_BASE(is_dir);
387 dlb2_port[response.id][DLB2_DIR_PORT].pp_addr =
388 (void *)(pp_base + (rte_mem_page_size() * response.id));
390 dlb2_port[response.id][DLB2_DIR_PORT].cq_base =
392 memset(&port_memory, 0, sizeof(port_memory));
394 dlb2_port[response.id][DLB2_DIR_PORT].mz = mz;
396 dlb2_list_init_head(&port_memory.list);
398 cfg->response = response;
404 rte_memzone_free(mz);
406 DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
413 dlb2_pf_dir_queue_create(struct dlb2_hw_dev *handle,
414 struct dlb2_create_dir_queue_args *cfg)
416 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
417 struct dlb2_cmd_response response = {0};
420 DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
422 ret = dlb2_pf_create_dir_queue(&dlb2_dev->hw,
427 cfg->response = response;
429 DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
436 dlb2_pf_map_qid(struct dlb2_hw_dev *handle,
437 struct dlb2_map_qid_args *cfg)
439 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
440 struct dlb2_cmd_response response = {0};
443 DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
445 ret = dlb2_hw_map_qid(&dlb2_dev->hw,
452 cfg->response = response;
454 DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
461 dlb2_pf_unmap_qid(struct dlb2_hw_dev *handle,
462 struct dlb2_unmap_qid_args *cfg)
464 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
465 struct dlb2_cmd_response response = {0};
468 DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
470 ret = dlb2_hw_unmap_qid(&dlb2_dev->hw,
477 cfg->response = response;
479 DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
486 dlb2_pf_pending_port_unmaps(struct dlb2_hw_dev *handle,
487 struct dlb2_pending_port_unmaps_args *args)
489 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
490 struct dlb2_cmd_response response = {0};
493 DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
495 ret = dlb2_hw_pending_port_unmaps(&dlb2_dev->hw,
502 args->response = response;
504 DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
511 dlb2_pf_sched_domain_start(struct dlb2_hw_dev *handle,
512 struct dlb2_start_domain_args *cfg)
514 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
515 struct dlb2_cmd_response response = {0};
518 DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
520 ret = dlb2_pf_start_domain(&dlb2_dev->hw,
525 cfg->response = response;
527 DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
534 dlb2_pf_get_ldb_queue_depth(struct dlb2_hw_dev *handle,
535 struct dlb2_get_ldb_queue_depth_args *args)
537 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
538 struct dlb2_cmd_response response = {0};
541 DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
543 ret = dlb2_hw_get_ldb_queue_depth(&dlb2_dev->hw,
550 args->response = response;
552 DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
559 dlb2_pf_get_dir_queue_depth(struct dlb2_hw_dev *handle,
560 struct dlb2_get_dir_queue_depth_args *args)
562 struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
563 struct dlb2_cmd_response response = {0};
566 DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
568 ret = dlb2_hw_get_dir_queue_depth(&dlb2_dev->hw,
575 args->response = response;
577 DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
584 dlb2_pf_iface_fn_ptrs_init(void)
586 dlb2_iface_low_level_io_init = dlb2_pf_low_level_io_init;
587 dlb2_iface_open = dlb2_pf_open;
588 dlb2_iface_domain_reset = dlb2_pf_domain_reset;
589 dlb2_iface_get_device_version = dlb2_pf_get_device_version;
590 dlb2_iface_hardware_init = dlb2_pf_hardware_init;
591 dlb2_iface_get_num_resources = dlb2_pf_get_num_resources;
592 dlb2_iface_get_cq_poll_mode = dlb2_pf_get_cq_poll_mode;
593 dlb2_iface_sched_domain_create = dlb2_pf_sched_domain_create;
594 dlb2_iface_ldb_queue_create = dlb2_pf_ldb_queue_create;
595 dlb2_iface_ldb_port_create = dlb2_pf_ldb_port_create;
596 dlb2_iface_dir_queue_create = dlb2_pf_dir_queue_create;
597 dlb2_iface_dir_port_create = dlb2_pf_dir_port_create;
598 dlb2_iface_map_qid = dlb2_pf_map_qid;
599 dlb2_iface_unmap_qid = dlb2_pf_unmap_qid;
600 dlb2_iface_get_ldb_queue_depth = dlb2_pf_get_ldb_queue_depth;
601 dlb2_iface_get_dir_queue_depth = dlb2_pf_get_dir_queue_depth;
602 dlb2_iface_sched_domain_start = dlb2_pf_sched_domain_start;
603 dlb2_iface_pending_port_unmaps = dlb2_pf_pending_port_unmaps;
604 dlb2_iface_get_sn_allocation = dlb2_pf_get_sn_allocation;
605 dlb2_iface_set_sn_allocation = dlb2_pf_set_sn_allocation;
606 dlb2_iface_get_sn_occupancy = dlb2_pf_get_sn_occupancy;
611 dlb2_eventdev_pci_init(struct rte_eventdev *eventdev)
614 struct rte_pci_device *pci_dev;
615 struct dlb2_devargs dlb2_args = {
616 .socket_id = rte_socket_id(),
617 .max_num_events = DLB2_MAX_NUM_LDB_CREDITS,
618 .num_dir_credits_override = -1,
619 .qid_depth_thresholds = { {0} },
620 .cos_id = DLB2_COS_DEFAULT
622 struct dlb2_eventdev *dlb2;
624 DLB2_LOG_DBG("Enter with dev_id=%d socket_id=%d",
625 eventdev->data->dev_id, eventdev->data->socket_id);
627 dlb2_pf_iface_fn_ptrs_init();
629 pci_dev = RTE_DEV_TO_PCI(eventdev->dev);
631 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
632 dlb2 = dlb2_pmd_priv(eventdev); /* rte_zmalloc_socket mem */
633 dlb2->version = DLB2_HW_DEVICE_FROM_PCI_ID(pci_dev);
635 /* Probe the DLB2 PF layer */
636 dlb2->qm_instance.pf_dev = dlb2_probe(pci_dev);
638 if (dlb2->qm_instance.pf_dev == NULL) {
639 DLB2_LOG_ERR("DLB2 PF Probe failed with error %d\n",
642 goto dlb2_probe_failed;
645 /* Were we invoked with runtime parameters? */
646 if (pci_dev->device.devargs) {
647 ret = dlb2_parse_params(pci_dev->device.devargs->args,
648 pci_dev->device.devargs->name,
652 DLB2_LOG_ERR("PFPMD failed to parse args ret=%d, errno=%d\n",
654 goto dlb2_probe_failed;
658 ret = dlb2_primary_eventdev_probe(eventdev,
662 dlb2 = dlb2_pmd_priv(eventdev);
663 dlb2->version = DLB2_HW_DEVICE_FROM_PCI_ID(pci_dev);
664 ret = dlb2_secondary_eventdev_probe(eventdev,
668 goto dlb2_probe_failed;
670 DLB2_LOG_INFO("DLB2 PF Probe success\n");
676 DLB2_LOG_INFO("DLB2 PF Probe failed, ret=%d\n", ret);
681 #define EVENTDEV_INTEL_VENDOR_ID 0x8086
683 static const struct rte_pci_id pci_id_dlb2_map[] = {
685 RTE_PCI_DEVICE(EVENTDEV_INTEL_VENDOR_ID,
686 PCI_DEVICE_ID_INTEL_DLB2_PF)
693 static const struct rte_pci_id pci_id_dlb2_5_map[] = {
695 RTE_PCI_DEVICE(EVENTDEV_INTEL_VENDOR_ID,
696 PCI_DEVICE_ID_INTEL_DLB2_5_PF)
704 event_dlb2_pci_probe(struct rte_pci_driver *pci_drv,
705 struct rte_pci_device *pci_dev)
709 ret = rte_event_pmd_pci_probe_named(pci_drv, pci_dev,
710 sizeof(struct dlb2_eventdev),
711 dlb2_eventdev_pci_init,
714 DLB2_LOG_INFO("rte_event_pmd_pci_probe_named() failed, "
722 event_dlb2_pci_remove(struct rte_pci_device *pci_dev)
726 ret = rte_event_pmd_pci_remove(pci_dev, NULL);
729 DLB2_LOG_INFO("rte_event_pmd_pci_remove() failed, "
738 event_dlb2_5_pci_probe(struct rte_pci_driver *pci_drv,
739 struct rte_pci_device *pci_dev)
743 ret = rte_event_pmd_pci_probe_named(pci_drv, pci_dev,
744 sizeof(struct dlb2_eventdev),
745 dlb2_eventdev_pci_init,
748 DLB2_LOG_INFO("rte_event_pmd_pci_probe_named() failed, "
756 event_dlb2_5_pci_remove(struct rte_pci_device *pci_dev)
760 ret = rte_event_pmd_pci_remove(pci_dev, NULL);
763 DLB2_LOG_INFO("rte_event_pmd_pci_remove() failed, "
771 static struct rte_pci_driver pci_eventdev_dlb2_pmd = {
772 .id_table = pci_id_dlb2_map,
773 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
774 .probe = event_dlb2_pci_probe,
775 .remove = event_dlb2_pci_remove,
778 static struct rte_pci_driver pci_eventdev_dlb2_5_pmd = {
779 .id_table = pci_id_dlb2_5_map,
780 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
781 .probe = event_dlb2_5_pci_probe,
782 .remove = event_dlb2_5_pci_remove,
785 RTE_PMD_REGISTER_PCI(event_dlb2_pf, pci_eventdev_dlb2_pmd);
786 RTE_PMD_REGISTER_PCI_TABLE(event_dlb2_pf, pci_id_dlb2_map);
788 RTE_PMD_REGISTER_PCI(event_dlb2_5_pf, pci_eventdev_dlb2_5_pmd);
789 RTE_PMD_REGISTER_PCI_TABLE(event_dlb2_5_pf, pci_id_dlb2_5_map);