1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 HiSilicon Limited
3 * Copyright(c) 2021 Intel Corporation
11 #include <rte_malloc.h>
12 #include <rte_memzone.h>
13 #include <rte_string_fns.h>
15 #include "rte_dmadev.h"
16 #include "rte_dmadev_pmd.h"
18 static int16_t dma_devices_max;
20 struct rte_dma_fp_object *rte_dma_fp_objs;
21 struct rte_dma_dev *rte_dma_devices;
23 /* Hold the dev_max information of the primary process. This field is
24 * set by the primary process and is read by the secondary process.
27 struct rte_dma_dev_data data[0];
28 } *dma_devices_shared_data;
30 RTE_LOG_REGISTER_DEFAULT(rte_dma_logtype, INFO);
31 #define RTE_DMA_LOG(level, ...) \
32 rte_log(RTE_LOG_ ## level, rte_dma_logtype, RTE_FMT("dma: " \
33 RTE_FMT_HEAD(__VA_ARGS__,) "\n", RTE_FMT_TAIL(__VA_ARGS__,)))
36 rte_dma_dev_max(size_t dev_max)
38 /* This function may be called before rte_eal_init(), so no rte library
39 * function can be called in this function.
41 if (dev_max == 0 || dev_max > INT16_MAX)
44 if (dma_devices_max > 0)
47 dma_devices_max = dev_max;
53 rte_dma_next_dev(int16_t start_dev_id)
55 int16_t dev_id = start_dev_id;
56 while (dev_id < dma_devices_max && rte_dma_devices[dev_id].state == RTE_DMA_DEV_UNUSED)
59 if (dev_id < dma_devices_max)
66 dma_check_name(const char *name)
71 RTE_DMA_LOG(ERR, "Name can't be NULL");
75 name_len = strnlen(name, RTE_DEV_NAME_MAX_LEN);
77 RTE_DMA_LOG(ERR, "Zero length DMA device name");
80 if (name_len >= RTE_DEV_NAME_MAX_LEN) {
81 RTE_DMA_LOG(ERR, "DMA device name is too long");
89 dma_find_free_id(void)
93 if (rte_dma_devices == NULL || dma_devices_shared_data == NULL)
96 for (i = 0; i < dma_devices_max; i++) {
97 if (dma_devices_shared_data->data[i].dev_name[0] == '\0')
104 static struct rte_dma_dev*
105 dma_find_by_name(const char *name)
109 if (rte_dma_devices == NULL)
112 for (i = 0; i < dma_devices_max; i++) {
113 if ((rte_dma_devices[i].state != RTE_DMA_DEV_UNUSED) &&
114 (!strcmp(name, rte_dma_devices[i].data->dev_name)))
115 return &rte_dma_devices[i];
121 static void dma_fp_object_dummy(struct rte_dma_fp_object *obj);
124 dma_fp_data_prepare(void)
130 if (rte_dma_fp_objs != NULL)
133 /* Fast-path object must align cacheline, but the return value of malloc
134 * may not be aligned to the cache line. Therefore, extra memory is
135 * applied for realignment.
136 * note: We do not call posix_memalign/aligned_alloc because it is
137 * version dependent on libc.
139 size = dma_devices_max * sizeof(struct rte_dma_fp_object) +
144 memset(ptr, 0, size);
146 rte_dma_fp_objs = RTE_PTR_ALIGN(ptr, RTE_CACHE_LINE_SIZE);
147 for (i = 0; i < dma_devices_max; i++)
148 dma_fp_object_dummy(&rte_dma_fp_objs[i]);
154 dma_dev_data_prepare(void)
158 if (rte_dma_devices != NULL)
161 size = dma_devices_max * sizeof(struct rte_dma_dev);
162 rte_dma_devices = malloc(size);
163 if (rte_dma_devices == NULL)
165 memset(rte_dma_devices, 0, size);
171 dma_shared_data_prepare(void)
173 const char *mz_name = "rte_dma_dev_data";
174 const struct rte_memzone *mz;
177 if (dma_devices_shared_data != NULL)
180 size = sizeof(*dma_devices_shared_data) +
181 sizeof(struct rte_dma_dev_data) * dma_devices_max;
183 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
184 mz = rte_memzone_reserve(mz_name, size, rte_socket_id(), 0);
186 mz = rte_memzone_lookup(mz_name);
190 dma_devices_shared_data = mz->addr;
191 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
192 memset(dma_devices_shared_data, 0, size);
193 dma_devices_shared_data->dev_max = dma_devices_max;
195 dma_devices_max = dma_devices_shared_data->dev_max;
202 dma_data_prepare(void)
206 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
207 if (dma_devices_max == 0)
208 dma_devices_max = RTE_DMADEV_DEFAULT_MAX;
209 ret = dma_fp_data_prepare();
212 ret = dma_dev_data_prepare();
215 ret = dma_shared_data_prepare();
219 ret = dma_shared_data_prepare();
222 ret = dma_fp_data_prepare();
225 ret = dma_dev_data_prepare();
233 static struct rte_dma_dev *
234 dma_allocate_primary(const char *name, int numa_node, size_t private_data_size)
236 struct rte_dma_dev *dev;
241 ret = dma_data_prepare();
243 RTE_DMA_LOG(ERR, "Cannot initialize dmadevs data");
247 dev = dma_find_by_name(name);
249 RTE_DMA_LOG(ERR, "DMA device already allocated");
253 dev_private = rte_zmalloc_socket(name, private_data_size,
254 RTE_CACHE_LINE_SIZE, numa_node);
255 if (dev_private == NULL) {
256 RTE_DMA_LOG(ERR, "Cannot allocate private data");
260 dev_id = dma_find_free_id();
262 RTE_DMA_LOG(ERR, "Reached maximum number of DMA devices");
263 rte_free(dev_private);
267 dev = &rte_dma_devices[dev_id];
268 dev->data = &dma_devices_shared_data->data[dev_id];
269 rte_strscpy(dev->data->dev_name, name, sizeof(dev->data->dev_name));
270 dev->data->dev_id = dev_id;
271 dev->data->numa_node = numa_node;
272 dev->data->dev_private = dev_private;
277 static struct rte_dma_dev *
278 dma_attach_secondary(const char *name)
280 struct rte_dma_dev *dev;
284 ret = dma_data_prepare();
286 RTE_DMA_LOG(ERR, "Cannot initialize dmadevs data");
290 for (i = 0; i < dma_devices_max; i++) {
291 if (!strcmp(dma_devices_shared_data->data[i].dev_name, name))
294 if (i == dma_devices_max) {
296 "Device %s is not driven by the primary process",
301 dev = &rte_dma_devices[i];
302 dev->data = &dma_devices_shared_data->data[i];
307 static struct rte_dma_dev *
308 dma_allocate(const char *name, int numa_node, size_t private_data_size)
310 struct rte_dma_dev *dev;
312 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
313 dev = dma_allocate_primary(name, numa_node, private_data_size);
315 dev = dma_attach_secondary(name);
318 dev->fp_obj = &rte_dma_fp_objs[dev->data->dev_id];
319 dma_fp_object_dummy(dev->fp_obj);
326 dma_release(struct rte_dma_dev *dev)
328 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
329 rte_free(dev->data->dev_private);
330 memset(dev->data, 0, sizeof(struct rte_dma_dev_data));
333 dma_fp_object_dummy(dev->fp_obj);
334 memset(dev, 0, sizeof(struct rte_dma_dev));
338 rte_dma_pmd_allocate(const char *name, int numa_node, size_t private_data_size)
340 struct rte_dma_dev *dev;
342 if (dma_check_name(name) != 0 || private_data_size == 0)
345 dev = dma_allocate(name, numa_node, private_data_size);
349 dev->state = RTE_DMA_DEV_REGISTERED;
355 rte_dma_pmd_release(const char *name)
357 struct rte_dma_dev *dev;
359 if (dma_check_name(name) != 0)
362 dev = dma_find_by_name(name);
366 if (dev->state == RTE_DMA_DEV_READY)
367 return rte_dma_close(dev->data->dev_id);
374 rte_dma_get_dev_id_by_name(const char *name)
376 struct rte_dma_dev *dev;
378 if (dma_check_name(name) != 0)
381 dev = dma_find_by_name(name);
385 return dev->data->dev_id;
389 rte_dma_is_valid(int16_t dev_id)
391 return (dev_id >= 0) && (dev_id < dma_devices_max) &&
392 rte_dma_devices != NULL &&
393 rte_dma_devices[dev_id].state != RTE_DMA_DEV_UNUSED;
397 rte_dma_count_avail(void)
402 if (rte_dma_devices == NULL)
405 for (i = 0; i < dma_devices_max; i++) {
406 if (rte_dma_devices[i].state != RTE_DMA_DEV_UNUSED)
414 rte_dma_info_get(int16_t dev_id, struct rte_dma_info *dev_info)
416 const struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
419 if (!rte_dma_is_valid(dev_id) || dev_info == NULL)
422 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_info_get, -ENOTSUP);
423 memset(dev_info, 0, sizeof(struct rte_dma_info));
424 ret = (*dev->dev_ops->dev_info_get)(dev, dev_info,
425 sizeof(struct rte_dma_info));
429 dev_info->dev_name = dev->data->dev_name;
430 dev_info->numa_node = dev->device->numa_node;
431 dev_info->nb_vchans = dev->data->dev_conf.nb_vchans;
437 rte_dma_configure(int16_t dev_id, const struct rte_dma_conf *dev_conf)
439 struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
440 struct rte_dma_info dev_info;
443 if (!rte_dma_is_valid(dev_id) || dev_conf == NULL)
446 if (dev->data->dev_started != 0) {
448 "Device %d must be stopped to allow configuration",
453 ret = rte_dma_info_get(dev_id, &dev_info);
455 RTE_DMA_LOG(ERR, "Device %d get device info fail", dev_id);
458 if (dev_conf->nb_vchans == 0) {
460 "Device %d configure zero vchans", dev_id);
463 if (dev_conf->nb_vchans > dev_info.max_vchans) {
465 "Device %d configure too many vchans", dev_id);
468 if (dev_conf->enable_silent &&
469 !(dev_info.dev_capa & RTE_DMA_CAPA_SILENT)) {
470 RTE_DMA_LOG(ERR, "Device %d don't support silent", dev_id);
474 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
475 ret = (*dev->dev_ops->dev_configure)(dev, dev_conf,
476 sizeof(struct rte_dma_conf));
478 memcpy(&dev->data->dev_conf, dev_conf,
479 sizeof(struct rte_dma_conf));
485 rte_dma_start(int16_t dev_id)
487 struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
490 if (!rte_dma_is_valid(dev_id))
493 if (dev->data->dev_conf.nb_vchans == 0) {
494 RTE_DMA_LOG(ERR, "Device %d must be configured first", dev_id);
498 if (dev->data->dev_started != 0) {
499 RTE_DMA_LOG(WARNING, "Device %d already started", dev_id);
503 if (dev->dev_ops->dev_start == NULL)
506 ret = (*dev->dev_ops->dev_start)(dev);
511 dev->data->dev_started = 1;
516 rte_dma_stop(int16_t dev_id)
518 struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
521 if (!rte_dma_is_valid(dev_id))
524 if (dev->data->dev_started == 0) {
525 RTE_DMA_LOG(WARNING, "Device %d already stopped", dev_id);
529 if (dev->dev_ops->dev_stop == NULL)
532 ret = (*dev->dev_ops->dev_stop)(dev);
537 dev->data->dev_started = 0;
542 rte_dma_close(int16_t dev_id)
544 struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
547 if (!rte_dma_is_valid(dev_id))
550 /* Device must be stopped before it can be closed */
551 if (dev->data->dev_started == 1) {
553 "Device %d must be stopped before closing", dev_id);
557 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP);
558 ret = (*dev->dev_ops->dev_close)(dev);
566 rte_dma_vchan_setup(int16_t dev_id, uint16_t vchan,
567 const struct rte_dma_vchan_conf *conf)
569 struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
570 struct rte_dma_info dev_info;
571 bool src_is_dev, dst_is_dev;
574 if (!rte_dma_is_valid(dev_id) || conf == NULL)
577 if (dev->data->dev_started != 0) {
579 "Device %d must be stopped to allow configuration",
584 ret = rte_dma_info_get(dev_id, &dev_info);
586 RTE_DMA_LOG(ERR, "Device %d get device info fail", dev_id);
589 if (dev->data->dev_conf.nb_vchans == 0) {
590 RTE_DMA_LOG(ERR, "Device %d must be configured first", dev_id);
593 if (vchan >= dev_info.nb_vchans) {
594 RTE_DMA_LOG(ERR, "Device %d vchan out range!", dev_id);
597 if (conf->direction != RTE_DMA_DIR_MEM_TO_MEM &&
598 conf->direction != RTE_DMA_DIR_MEM_TO_DEV &&
599 conf->direction != RTE_DMA_DIR_DEV_TO_MEM &&
600 conf->direction != RTE_DMA_DIR_DEV_TO_DEV) {
601 RTE_DMA_LOG(ERR, "Device %d direction invalid!", dev_id);
604 if (conf->direction == RTE_DMA_DIR_MEM_TO_MEM &&
605 !(dev_info.dev_capa & RTE_DMA_CAPA_MEM_TO_MEM)) {
607 "Device %d don't support mem2mem transfer", dev_id);
610 if (conf->direction == RTE_DMA_DIR_MEM_TO_DEV &&
611 !(dev_info.dev_capa & RTE_DMA_CAPA_MEM_TO_DEV)) {
613 "Device %d don't support mem2dev transfer", dev_id);
616 if (conf->direction == RTE_DMA_DIR_DEV_TO_MEM &&
617 !(dev_info.dev_capa & RTE_DMA_CAPA_DEV_TO_MEM)) {
619 "Device %d don't support dev2mem transfer", dev_id);
622 if (conf->direction == RTE_DMA_DIR_DEV_TO_DEV &&
623 !(dev_info.dev_capa & RTE_DMA_CAPA_DEV_TO_DEV)) {
625 "Device %d don't support dev2dev transfer", dev_id);
628 if (conf->nb_desc < dev_info.min_desc ||
629 conf->nb_desc > dev_info.max_desc) {
631 "Device %d number of descriptors invalid", dev_id);
634 src_is_dev = conf->direction == RTE_DMA_DIR_DEV_TO_MEM ||
635 conf->direction == RTE_DMA_DIR_DEV_TO_DEV;
636 if ((conf->src_port.port_type == RTE_DMA_PORT_NONE && src_is_dev) ||
637 (conf->src_port.port_type != RTE_DMA_PORT_NONE && !src_is_dev)) {
638 RTE_DMA_LOG(ERR, "Device %d source port type invalid", dev_id);
641 dst_is_dev = conf->direction == RTE_DMA_DIR_MEM_TO_DEV ||
642 conf->direction == RTE_DMA_DIR_DEV_TO_DEV;
643 if ((conf->dst_port.port_type == RTE_DMA_PORT_NONE && dst_is_dev) ||
644 (conf->dst_port.port_type != RTE_DMA_PORT_NONE && !dst_is_dev)) {
646 "Device %d destination port type invalid", dev_id);
650 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vchan_setup, -ENOTSUP);
651 return (*dev->dev_ops->vchan_setup)(dev, vchan, conf,
652 sizeof(struct rte_dma_vchan_conf));
656 rte_dma_stats_get(int16_t dev_id, uint16_t vchan, struct rte_dma_stats *stats)
658 const struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
660 if (!rte_dma_is_valid(dev_id) || stats == NULL)
663 if (vchan >= dev->data->dev_conf.nb_vchans &&
664 vchan != RTE_DMA_ALL_VCHAN) {
666 "Device %d vchan %u out of range", dev_id, vchan);
670 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
671 memset(stats, 0, sizeof(struct rte_dma_stats));
672 return (*dev->dev_ops->stats_get)(dev, vchan, stats,
673 sizeof(struct rte_dma_stats));
677 rte_dma_stats_reset(int16_t dev_id, uint16_t vchan)
679 struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
681 if (!rte_dma_is_valid(dev_id))
684 if (vchan >= dev->data->dev_conf.nb_vchans &&
685 vchan != RTE_DMA_ALL_VCHAN) {
687 "Device %d vchan %u out of range", dev_id, vchan);
691 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_reset, -ENOTSUP);
692 return (*dev->dev_ops->stats_reset)(dev, vchan);
696 rte_dma_vchan_status(int16_t dev_id, uint16_t vchan, enum rte_dma_vchan_status *status)
698 struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
700 if (!rte_dma_is_valid(dev_id))
703 if (vchan >= dev->data->dev_conf.nb_vchans) {
704 RTE_DMA_LOG(ERR, "Device %u vchan %u out of range\n", dev_id, vchan);
708 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vchan_status, -ENOTSUP);
709 return (*dev->dev_ops->vchan_status)(dev, vchan, status);
713 dma_capability_name(uint64_t capability)
715 static const struct {
719 { RTE_DMA_CAPA_MEM_TO_MEM, "mem2mem" },
720 { RTE_DMA_CAPA_MEM_TO_DEV, "mem2dev" },
721 { RTE_DMA_CAPA_DEV_TO_MEM, "dev2mem" },
722 { RTE_DMA_CAPA_DEV_TO_DEV, "dev2dev" },
723 { RTE_DMA_CAPA_SVA, "sva" },
724 { RTE_DMA_CAPA_SILENT, "silent" },
725 { RTE_DMA_CAPA_HANDLES_ERRORS, "handles_errors" },
726 { RTE_DMA_CAPA_OPS_COPY, "copy" },
727 { RTE_DMA_CAPA_OPS_COPY_SG, "copy_sg" },
728 { RTE_DMA_CAPA_OPS_FILL, "fill" },
731 const char *name = "unknown";
734 for (i = 0; i < RTE_DIM(capa_names); i++) {
735 if (capability == capa_names[i].capability) {
736 name = capa_names[i].name;
745 dma_dump_capability(FILE *f, uint64_t dev_capa)
749 (void)fprintf(f, " dev_capa: 0x%" PRIx64 " -", dev_capa);
750 while (dev_capa > 0) {
751 capa = 1ull << __builtin_ctzll(dev_capa);
752 (void)fprintf(f, " %s", dma_capability_name(capa));
755 (void)fprintf(f, "\n");
759 rte_dma_dump(int16_t dev_id, FILE *f)
761 const struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
762 struct rte_dma_info dev_info;
765 if (!rte_dma_is_valid(dev_id) || f == NULL)
768 ret = rte_dma_info_get(dev_id, &dev_info);
770 RTE_DMA_LOG(ERR, "Device %d get device info fail", dev_id);
774 (void)fprintf(f, "DMA Dev %d, '%s' [%s]\n",
777 dev->data->dev_started ? "started" : "stopped");
778 dma_dump_capability(f, dev_info.dev_capa);
779 (void)fprintf(f, " max_vchans_supported: %u\n", dev_info.max_vchans);
780 (void)fprintf(f, " nb_vchans_configured: %u\n", dev_info.nb_vchans);
781 (void)fprintf(f, " silent_mode: %s\n",
782 dev->data->dev_conf.enable_silent ? "on" : "off");
784 if (dev->dev_ops->dev_dump != NULL)
785 return (*dev->dev_ops->dev_dump)(dev, f);
791 dummy_copy(__rte_unused void *dev_private, __rte_unused uint16_t vchan,
792 __rte_unused rte_iova_t src, __rte_unused rte_iova_t dst,
793 __rte_unused uint32_t length, __rte_unused uint64_t flags)
795 RTE_DMA_LOG(ERR, "copy is not configured or not supported.");
800 dummy_copy_sg(__rte_unused void *dev_private, __rte_unused uint16_t vchan,
801 __rte_unused const struct rte_dma_sge *src,
802 __rte_unused const struct rte_dma_sge *dst,
803 __rte_unused uint16_t nb_src, __rte_unused uint16_t nb_dst,
804 __rte_unused uint64_t flags)
806 RTE_DMA_LOG(ERR, "copy_sg is not configured or not supported.");
811 dummy_fill(__rte_unused void *dev_private, __rte_unused uint16_t vchan,
812 __rte_unused uint64_t pattern, __rte_unused rte_iova_t dst,
813 __rte_unused uint32_t length, __rte_unused uint64_t flags)
815 RTE_DMA_LOG(ERR, "fill is not configured or not supported.");
820 dummy_submit(__rte_unused void *dev_private, __rte_unused uint16_t vchan)
822 RTE_DMA_LOG(ERR, "submit is not configured or not supported.");
827 dummy_completed(__rte_unused void *dev_private, __rte_unused uint16_t vchan,
828 __rte_unused const uint16_t nb_cpls,
829 __rte_unused uint16_t *last_idx, __rte_unused bool *has_error)
831 RTE_DMA_LOG(ERR, "completed is not configured or not supported.");
836 dummy_completed_status(__rte_unused void *dev_private,
837 __rte_unused uint16_t vchan,
838 __rte_unused const uint16_t nb_cpls,
839 __rte_unused uint16_t *last_idx,
840 __rte_unused enum rte_dma_status_code *status)
843 "completed_status is not configured or not supported.");
848 dummy_burst_capacity(__rte_unused const void *dev_private,
849 __rte_unused uint16_t vchan)
851 RTE_DMA_LOG(ERR, "burst_capacity is not configured or not supported.");
856 dma_fp_object_dummy(struct rte_dma_fp_object *obj)
858 obj->dev_private = NULL;
859 obj->copy = dummy_copy;
860 obj->copy_sg = dummy_copy_sg;
861 obj->fill = dummy_fill;
862 obj->submit = dummy_submit;
863 obj->completed = dummy_completed;
864 obj->completed_status = dummy_completed_status;
865 obj->burst_capacity = dummy_burst_capacity;