1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 HiSilicon Limited
3 * Copyright(c) 2021 Intel Corporation
11 #include <rte_malloc.h>
12 #include <rte_memzone.h>
13 #include <rte_string_fns.h>
15 #include "rte_dmadev.h"
16 #include "rte_dmadev_pmd.h"
18 static int16_t dma_devices_max;
20 struct rte_dma_fp_object *rte_dma_fp_objs;
21 struct rte_dma_dev *rte_dma_devices;
23 /* Hold the dev_max information of the primary process. This field is
24 * set by the primary process and is read by the secondary process.
27 struct rte_dma_dev_data data[0];
28 } *dma_devices_shared_data;
30 RTE_LOG_REGISTER_DEFAULT(rte_dma_logtype, INFO);
31 #define RTE_DMA_LOG(level, ...) \
32 rte_log(RTE_LOG_ ## level, rte_dma_logtype, RTE_FMT("dma: " \
33 RTE_FMT_HEAD(__VA_ARGS__,) "\n", RTE_FMT_TAIL(__VA_ARGS__,)))
36 rte_dma_dev_max(size_t dev_max)
38 /* This function may be called before rte_eal_init(), so no rte library
39 * function can be called in this function.
41 if (dev_max == 0 || dev_max > INT16_MAX)
44 if (dma_devices_max > 0)
47 dma_devices_max = dev_max;
53 dma_check_name(const char *name)
58 RTE_DMA_LOG(ERR, "Name can't be NULL");
62 name_len = strnlen(name, RTE_DEV_NAME_MAX_LEN);
64 RTE_DMA_LOG(ERR, "Zero length DMA device name");
67 if (name_len >= RTE_DEV_NAME_MAX_LEN) {
68 RTE_DMA_LOG(ERR, "DMA device name is too long");
76 dma_find_free_id(void)
80 if (rte_dma_devices == NULL || dma_devices_shared_data == NULL)
83 for (i = 0; i < dma_devices_max; i++) {
84 if (dma_devices_shared_data->data[i].dev_name[0] == '\0')
91 static struct rte_dma_dev*
92 dma_find_by_name(const char *name)
96 if (rte_dma_devices == NULL)
99 for (i = 0; i < dma_devices_max; i++) {
100 if ((rte_dma_devices[i].state != RTE_DMA_DEV_UNUSED) &&
101 (!strcmp(name, rte_dma_devices[i].data->dev_name)))
102 return &rte_dma_devices[i];
108 static void dma_fp_object_dummy(struct rte_dma_fp_object *obj);
111 dma_fp_data_prepare(void)
117 if (rte_dma_fp_objs != NULL)
120 /* Fast-path object must align cacheline, but the return value of malloc
121 * may not be aligned to the cache line. Therefore, extra memory is
122 * applied for realignment.
123 * note: We do not call posix_memalign/aligned_alloc because it is
124 * version dependent on libc.
126 size = dma_devices_max * sizeof(struct rte_dma_fp_object) +
131 memset(ptr, 0, size);
133 rte_dma_fp_objs = RTE_PTR_ALIGN(ptr, RTE_CACHE_LINE_SIZE);
134 for (i = 0; i < dma_devices_max; i++)
135 dma_fp_object_dummy(&rte_dma_fp_objs[i]);
141 dma_dev_data_prepare(void)
145 if (rte_dma_devices != NULL)
148 size = dma_devices_max * sizeof(struct rte_dma_dev);
149 rte_dma_devices = malloc(size);
150 if (rte_dma_devices == NULL)
152 memset(rte_dma_devices, 0, size);
158 dma_shared_data_prepare(void)
160 const char *mz_name = "rte_dma_dev_data";
161 const struct rte_memzone *mz;
164 if (dma_devices_shared_data != NULL)
167 size = sizeof(*dma_devices_shared_data) +
168 sizeof(struct rte_dma_dev_data) * dma_devices_max;
170 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
171 mz = rte_memzone_reserve(mz_name, size, rte_socket_id(), 0);
173 mz = rte_memzone_lookup(mz_name);
177 dma_devices_shared_data = mz->addr;
178 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
179 memset(dma_devices_shared_data, 0, size);
180 dma_devices_shared_data->dev_max = dma_devices_max;
182 dma_devices_max = dma_devices_shared_data->dev_max;
189 dma_data_prepare(void)
193 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
194 if (dma_devices_max == 0)
195 dma_devices_max = RTE_DMADEV_DEFAULT_MAX;
196 ret = dma_fp_data_prepare();
199 ret = dma_dev_data_prepare();
202 ret = dma_shared_data_prepare();
206 ret = dma_shared_data_prepare();
209 ret = dma_fp_data_prepare();
212 ret = dma_dev_data_prepare();
220 static struct rte_dma_dev *
221 dma_allocate_primary(const char *name, int numa_node, size_t private_data_size)
223 struct rte_dma_dev *dev;
228 ret = dma_data_prepare();
230 RTE_DMA_LOG(ERR, "Cannot initialize dmadevs data");
234 dev = dma_find_by_name(name);
236 RTE_DMA_LOG(ERR, "DMA device already allocated");
240 dev_private = rte_zmalloc_socket(name, private_data_size,
241 RTE_CACHE_LINE_SIZE, numa_node);
242 if (dev_private == NULL) {
243 RTE_DMA_LOG(ERR, "Cannot allocate private data");
247 dev_id = dma_find_free_id();
249 RTE_DMA_LOG(ERR, "Reached maximum number of DMA devices");
250 rte_free(dev_private);
254 dev = &rte_dma_devices[dev_id];
255 dev->data = &dma_devices_shared_data->data[dev_id];
256 rte_strscpy(dev->data->dev_name, name, sizeof(dev->data->dev_name));
257 dev->data->dev_id = dev_id;
258 dev->data->numa_node = numa_node;
259 dev->data->dev_private = dev_private;
264 static struct rte_dma_dev *
265 dma_attach_secondary(const char *name)
267 struct rte_dma_dev *dev;
271 ret = dma_data_prepare();
273 RTE_DMA_LOG(ERR, "Cannot initialize dmadevs data");
277 for (i = 0; i < dma_devices_max; i++) {
278 if (!strcmp(dma_devices_shared_data->data[i].dev_name, name))
281 if (i == dma_devices_max) {
283 "Device %s is not driven by the primary process",
288 dev = &rte_dma_devices[i];
289 dev->data = &dma_devices_shared_data->data[i];
294 static struct rte_dma_dev *
295 dma_allocate(const char *name, int numa_node, size_t private_data_size)
297 struct rte_dma_dev *dev;
299 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
300 dev = dma_allocate_primary(name, numa_node, private_data_size);
302 dev = dma_attach_secondary(name);
305 dev->fp_obj = &rte_dma_fp_objs[dev->data->dev_id];
306 dma_fp_object_dummy(dev->fp_obj);
313 dma_release(struct rte_dma_dev *dev)
315 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
316 rte_free(dev->data->dev_private);
317 memset(dev->data, 0, sizeof(struct rte_dma_dev_data));
320 dma_fp_object_dummy(dev->fp_obj);
321 memset(dev, 0, sizeof(struct rte_dma_dev));
325 rte_dma_pmd_allocate(const char *name, int numa_node, size_t private_data_size)
327 struct rte_dma_dev *dev;
329 if (dma_check_name(name) != 0 || private_data_size == 0)
332 dev = dma_allocate(name, numa_node, private_data_size);
336 dev->state = RTE_DMA_DEV_REGISTERED;
342 rte_dma_pmd_release(const char *name)
344 struct rte_dma_dev *dev;
346 if (dma_check_name(name) != 0)
349 dev = dma_find_by_name(name);
353 if (dev->state == RTE_DMA_DEV_READY)
354 return rte_dma_close(dev->data->dev_id);
361 rte_dma_get_dev_id_by_name(const char *name)
363 struct rte_dma_dev *dev;
365 if (dma_check_name(name) != 0)
368 dev = dma_find_by_name(name);
372 return dev->data->dev_id;
376 rte_dma_is_valid(int16_t dev_id)
378 return (dev_id >= 0) && (dev_id < dma_devices_max) &&
379 rte_dma_devices != NULL &&
380 rte_dma_devices[dev_id].state != RTE_DMA_DEV_UNUSED;
384 rte_dma_count_avail(void)
389 if (rte_dma_devices == NULL)
392 for (i = 0; i < dma_devices_max; i++) {
393 if (rte_dma_devices[i].state != RTE_DMA_DEV_UNUSED)
401 rte_dma_info_get(int16_t dev_id, struct rte_dma_info *dev_info)
403 const struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
406 if (!rte_dma_is_valid(dev_id) || dev_info == NULL)
409 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_info_get, -ENOTSUP);
410 memset(dev_info, 0, sizeof(struct rte_dma_info));
411 ret = (*dev->dev_ops->dev_info_get)(dev, dev_info,
412 sizeof(struct rte_dma_info));
416 dev_info->dev_name = dev->data->dev_name;
417 dev_info->numa_node = dev->device->numa_node;
418 dev_info->nb_vchans = dev->data->dev_conf.nb_vchans;
424 rte_dma_configure(int16_t dev_id, const struct rte_dma_conf *dev_conf)
426 struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
427 struct rte_dma_info dev_info;
430 if (!rte_dma_is_valid(dev_id) || dev_conf == NULL)
433 if (dev->data->dev_started != 0) {
435 "Device %d must be stopped to allow configuration",
440 ret = rte_dma_info_get(dev_id, &dev_info);
442 RTE_DMA_LOG(ERR, "Device %d get device info fail", dev_id);
445 if (dev_conf->nb_vchans == 0) {
447 "Device %d configure zero vchans", dev_id);
450 if (dev_conf->nb_vchans > dev_info.max_vchans) {
452 "Device %d configure too many vchans", dev_id);
455 if (dev_conf->enable_silent &&
456 !(dev_info.dev_capa & RTE_DMA_CAPA_SILENT)) {
457 RTE_DMA_LOG(ERR, "Device %d don't support silent", dev_id);
461 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
462 ret = (*dev->dev_ops->dev_configure)(dev, dev_conf,
463 sizeof(struct rte_dma_conf));
465 memcpy(&dev->data->dev_conf, dev_conf,
466 sizeof(struct rte_dma_conf));
472 rte_dma_start(int16_t dev_id)
474 struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
477 if (!rte_dma_is_valid(dev_id))
480 if (dev->data->dev_conf.nb_vchans == 0) {
481 RTE_DMA_LOG(ERR, "Device %d must be configured first", dev_id);
485 if (dev->data->dev_started != 0) {
486 RTE_DMA_LOG(WARNING, "Device %d already started", dev_id);
490 if (dev->dev_ops->dev_start == NULL)
493 ret = (*dev->dev_ops->dev_start)(dev);
498 dev->data->dev_started = 1;
503 rte_dma_stop(int16_t dev_id)
505 struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
508 if (!rte_dma_is_valid(dev_id))
511 if (dev->data->dev_started == 0) {
512 RTE_DMA_LOG(WARNING, "Device %d already stopped", dev_id);
516 if (dev->dev_ops->dev_stop == NULL)
519 ret = (*dev->dev_ops->dev_stop)(dev);
524 dev->data->dev_started = 0;
529 rte_dma_close(int16_t dev_id)
531 struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
534 if (!rte_dma_is_valid(dev_id))
537 /* Device must be stopped before it can be closed */
538 if (dev->data->dev_started == 1) {
540 "Device %d must be stopped before closing", dev_id);
544 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP);
545 ret = (*dev->dev_ops->dev_close)(dev);
553 rte_dma_vchan_setup(int16_t dev_id, uint16_t vchan,
554 const struct rte_dma_vchan_conf *conf)
556 struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
557 struct rte_dma_info dev_info;
558 bool src_is_dev, dst_is_dev;
561 if (!rte_dma_is_valid(dev_id) || conf == NULL)
564 if (dev->data->dev_started != 0) {
566 "Device %d must be stopped to allow configuration",
571 ret = rte_dma_info_get(dev_id, &dev_info);
573 RTE_DMA_LOG(ERR, "Device %d get device info fail", dev_id);
576 if (dev->data->dev_conf.nb_vchans == 0) {
577 RTE_DMA_LOG(ERR, "Device %d must be configured first", dev_id);
580 if (vchan >= dev_info.nb_vchans) {
581 RTE_DMA_LOG(ERR, "Device %d vchan out range!", dev_id);
584 if (conf->direction != RTE_DMA_DIR_MEM_TO_MEM &&
585 conf->direction != RTE_DMA_DIR_MEM_TO_DEV &&
586 conf->direction != RTE_DMA_DIR_DEV_TO_MEM &&
587 conf->direction != RTE_DMA_DIR_DEV_TO_DEV) {
588 RTE_DMA_LOG(ERR, "Device %d direction invalid!", dev_id);
591 if (conf->direction == RTE_DMA_DIR_MEM_TO_MEM &&
592 !(dev_info.dev_capa & RTE_DMA_CAPA_MEM_TO_MEM)) {
594 "Device %d don't support mem2mem transfer", dev_id);
597 if (conf->direction == RTE_DMA_DIR_MEM_TO_DEV &&
598 !(dev_info.dev_capa & RTE_DMA_CAPA_MEM_TO_DEV)) {
600 "Device %d don't support mem2dev transfer", dev_id);
603 if (conf->direction == RTE_DMA_DIR_DEV_TO_MEM &&
604 !(dev_info.dev_capa & RTE_DMA_CAPA_DEV_TO_MEM)) {
606 "Device %d don't support dev2mem transfer", dev_id);
609 if (conf->direction == RTE_DMA_DIR_DEV_TO_DEV &&
610 !(dev_info.dev_capa & RTE_DMA_CAPA_DEV_TO_DEV)) {
612 "Device %d don't support dev2dev transfer", dev_id);
615 if (conf->nb_desc < dev_info.min_desc ||
616 conf->nb_desc > dev_info.max_desc) {
618 "Device %d number of descriptors invalid", dev_id);
621 src_is_dev = conf->direction == RTE_DMA_DIR_DEV_TO_MEM ||
622 conf->direction == RTE_DMA_DIR_DEV_TO_DEV;
623 if ((conf->src_port.port_type == RTE_DMA_PORT_NONE && src_is_dev) ||
624 (conf->src_port.port_type != RTE_DMA_PORT_NONE && !src_is_dev)) {
625 RTE_DMA_LOG(ERR, "Device %d source port type invalid", dev_id);
628 dst_is_dev = conf->direction == RTE_DMA_DIR_MEM_TO_DEV ||
629 conf->direction == RTE_DMA_DIR_DEV_TO_DEV;
630 if ((conf->dst_port.port_type == RTE_DMA_PORT_NONE && dst_is_dev) ||
631 (conf->dst_port.port_type != RTE_DMA_PORT_NONE && !dst_is_dev)) {
633 "Device %d destination port type invalid", dev_id);
637 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vchan_setup, -ENOTSUP);
638 return (*dev->dev_ops->vchan_setup)(dev, vchan, conf,
639 sizeof(struct rte_dma_vchan_conf));
643 rte_dma_stats_get(int16_t dev_id, uint16_t vchan, struct rte_dma_stats *stats)
645 const struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
647 if (!rte_dma_is_valid(dev_id) || stats == NULL)
650 if (vchan >= dev->data->dev_conf.nb_vchans &&
651 vchan != RTE_DMA_ALL_VCHAN) {
653 "Device %d vchan %u out of range", dev_id, vchan);
657 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
658 memset(stats, 0, sizeof(struct rte_dma_stats));
659 return (*dev->dev_ops->stats_get)(dev, vchan, stats,
660 sizeof(struct rte_dma_stats));
664 rte_dma_stats_reset(int16_t dev_id, uint16_t vchan)
666 struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
668 if (!rte_dma_is_valid(dev_id))
671 if (vchan >= dev->data->dev_conf.nb_vchans &&
672 vchan != RTE_DMA_ALL_VCHAN) {
674 "Device %d vchan %u out of range", dev_id, vchan);
678 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_reset, -ENOTSUP);
679 return (*dev->dev_ops->stats_reset)(dev, vchan);
683 dma_capability_name(uint64_t capability)
685 static const struct {
689 { RTE_DMA_CAPA_MEM_TO_MEM, "mem2mem" },
690 { RTE_DMA_CAPA_MEM_TO_DEV, "mem2dev" },
691 { RTE_DMA_CAPA_DEV_TO_MEM, "dev2mem" },
692 { RTE_DMA_CAPA_DEV_TO_DEV, "dev2dev" },
693 { RTE_DMA_CAPA_SVA, "sva" },
694 { RTE_DMA_CAPA_SILENT, "silent" },
695 { RTE_DMA_CAPA_OPS_COPY, "copy" },
696 { RTE_DMA_CAPA_OPS_COPY_SG, "copy_sg" },
697 { RTE_DMA_CAPA_OPS_FILL, "fill" },
700 const char *name = "unknown";
703 for (i = 0; i < RTE_DIM(capa_names); i++) {
704 if (capability == capa_names[i].capability) {
705 name = capa_names[i].name;
714 dma_dump_capability(FILE *f, uint64_t dev_capa)
718 (void)fprintf(f, " dev_capa: 0x%" PRIx64 " -", dev_capa);
719 while (dev_capa > 0) {
720 capa = 1ull << __builtin_ctzll(dev_capa);
721 (void)fprintf(f, " %s", dma_capability_name(capa));
724 (void)fprintf(f, "\n");
728 rte_dma_dump(int16_t dev_id, FILE *f)
730 const struct rte_dma_dev *dev = &rte_dma_devices[dev_id];
731 struct rte_dma_info dev_info;
734 if (!rte_dma_is_valid(dev_id) || f == NULL)
737 ret = rte_dma_info_get(dev_id, &dev_info);
739 RTE_DMA_LOG(ERR, "Device %d get device info fail", dev_id);
743 (void)fprintf(f, "DMA Dev %d, '%s' [%s]\n",
746 dev->data->dev_started ? "started" : "stopped");
747 dma_dump_capability(f, dev_info.dev_capa);
748 (void)fprintf(f, " max_vchans_supported: %u\n", dev_info.max_vchans);
749 (void)fprintf(f, " nb_vchans_configured: %u\n", dev_info.nb_vchans);
750 (void)fprintf(f, " silent_mode: %s\n",
751 dev->data->dev_conf.enable_silent ? "on" : "off");
753 if (dev->dev_ops->dev_dump != NULL)
754 return (*dev->dev_ops->dev_dump)(dev, f);
760 dummy_copy(__rte_unused void *dev_private, __rte_unused uint16_t vchan,
761 __rte_unused rte_iova_t src, __rte_unused rte_iova_t dst,
762 __rte_unused uint32_t length, __rte_unused uint64_t flags)
764 RTE_DMA_LOG(ERR, "copy is not configured or not supported.");
769 dummy_copy_sg(__rte_unused void *dev_private, __rte_unused uint16_t vchan,
770 __rte_unused const struct rte_dma_sge *src,
771 __rte_unused const struct rte_dma_sge *dst,
772 __rte_unused uint16_t nb_src, __rte_unused uint16_t nb_dst,
773 __rte_unused uint64_t flags)
775 RTE_DMA_LOG(ERR, "copy_sg is not configured or not supported.");
780 dummy_fill(__rte_unused void *dev_private, __rte_unused uint16_t vchan,
781 __rte_unused uint64_t pattern, __rte_unused rte_iova_t dst,
782 __rte_unused uint32_t length, __rte_unused uint64_t flags)
784 RTE_DMA_LOG(ERR, "fill is not configured or not supported.");
789 dummy_submit(__rte_unused void *dev_private, __rte_unused uint16_t vchan)
791 RTE_DMA_LOG(ERR, "submit is not configured or not supported.");
796 dummy_completed(__rte_unused void *dev_private, __rte_unused uint16_t vchan,
797 __rte_unused const uint16_t nb_cpls,
798 __rte_unused uint16_t *last_idx, __rte_unused bool *has_error)
800 RTE_DMA_LOG(ERR, "completed is not configured or not supported.");
805 dummy_completed_status(__rte_unused void *dev_private,
806 __rte_unused uint16_t vchan,
807 __rte_unused const uint16_t nb_cpls,
808 __rte_unused uint16_t *last_idx,
809 __rte_unused enum rte_dma_status_code *status)
812 "completed_status is not configured or not supported.");
817 dma_fp_object_dummy(struct rte_dma_fp_object *obj)
819 obj->dev_private = NULL;
820 obj->copy = dummy_copy;
821 obj->copy_sg = dummy_copy_sg;
822 obj->fill = dummy_fill;
823 obj->submit = dummy_submit;
824 obj->completed = dummy_completed;
825 obj->completed_status = dummy_completed_status;