From c5eebf85badc34b1fda702f087200ce934f31ec1 Mon Sep 17 00:00:00 2001 From: Xiaoyun Li Date: Tue, 2 Jul 2019 14:25:21 +0800 Subject: [PATCH] examples/ntb: add example for NTB Enable an example for rawdev ntb. Support interactive mode to send file on one host and receive file from another host. The command line would be 'send [filepath]' and 'receive [filepath]'. But since the FIFO is not enabled right now, use rte_memcpy as the enqueue and dequeue functions and only support transmitting file no more than 4M. Signed-off-by: Xiaoyun Li Acked-by: Jingjing Wu Reviewed-by: Xiaolong Ye --- MAINTAINERS | 2 + doc/guides/sample_app_ug/index.rst | 1 + doc/guides/sample_app_ug/ntb.rst | 47 ++++ drivers/raw/ntb/ntb.c | 28 ++- examples/Makefile | 1 + examples/meson.build | 2 +- examples/ntb/Makefile | 68 ++++++ examples/ntb/meson.build | 16 ++ examples/ntb/ntb_fwd.c | 377 +++++++++++++++++++++++++++++ 9 files changed, 533 insertions(+), 9 deletions(-) create mode 100644 doc/guides/sample_app_ug/ntb.rst create mode 100644 examples/ntb/Makefile create mode 100644 examples/ntb/meson.build create mode 100644 examples/ntb/ntb_fwd.c diff --git a/MAINTAINERS b/MAINTAINERS index 4aba96a374..3e29c57c1f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1116,6 +1116,8 @@ M: Xiaoyun Li M: Jingjing Wu F: drivers/raw/ntb/ F: doc/guides/rawdevs/ntb.rst +F: examples/ntb/ +F: doc/guides/sample_app_ug/ntb.rst Packet processing diff --git a/doc/guides/sample_app_ug/index.rst b/doc/guides/sample_app_ug/index.rst index 2945be08f3..f23f8f59e9 100644 --- a/doc/guides/sample_app_ug/index.rst +++ b/doc/guides/sample_app_ug/index.rst @@ -58,3 +58,4 @@ Sample Applications User Guides fips_validation ipsec_secgw bbdev_app + ntb diff --git a/doc/guides/sample_app_ug/ntb.rst b/doc/guides/sample_app_ug/ntb.rst new file mode 100644 index 0000000000..079242175b --- /dev/null +++ b/doc/guides/sample_app_ug/ntb.rst @@ -0,0 +1,47 @@ +.. SPDX-License-Identifier: BSD-3-Clause + Copyright(c) 2019 Intel Corporation. + +NTB Sample Application +====================== + +The ntb sample application shows how to use ntb rawdev driver. +This sample provides interactive mode to transmit file between +two hosts. + +Compiling the Application +------------------------- + +To compile the sample application see :doc:`compiling`. + +The application is located in the ``ntb`` sub-directory. + +Running the Application +----------------------- + +The application requires an available core for each port, plus one. +The only available options are the standard ones for the EAL: + +.. code-block:: console + + ./build/ntb_fwd -c 0xf -n 6 -- -i + +Refer to the *DPDK Getting Started Guide* for general information on +running applications and the Environment Abstraction Layer (EAL) +options. + +Using the application +--------------------- + +The application is console-driven using the cmdline DPDK interface: + +.. code-block:: console + + ntb> + +From this interface the available commands and descriptions of what +they do as as follows: + +* ``send [filepath]``: Send file to the peer host. +* ``receive [filepath]``: Receive file to [filepath]. Need the peer + to send file successfully first. +* ``quit``: Exit program diff --git a/drivers/raw/ntb/ntb.c b/drivers/raw/ntb/ntb.c index c83b8d9642..4ba2f3a384 100644 --- a/drivers/raw/ntb/ntb.c +++ b/drivers/raw/ntb/ntb.c @@ -240,11 +240,19 @@ ntb_enqueue_bufs(struct rte_rawdev *dev, unsigned int count, rte_rawdev_obj_t context) { - RTE_SET_USED(dev); - RTE_SET_USED(buffers); - RTE_SET_USED(count); - RTE_SET_USED(context); + /* Not FIFO right now. Just for testing memory write. */ + struct ntb_hw *hw = dev->dev_private; + unsigned int i; + void *bar_addr; + size_t size; + + if (hw->ntb_ops->get_peer_mw_addr == NULL) + return -ENOTSUP; + bar_addr = (*hw->ntb_ops->get_peer_mw_addr)(dev, 0); + size = (size_t)context; + for (i = 0; i < count; i++) + rte_memcpy(bar_addr, buffers[i]->buf_addr, size); return 0; } @@ -254,11 +262,15 @@ ntb_dequeue_bufs(struct rte_rawdev *dev, unsigned int count, rte_rawdev_obj_t context) { - RTE_SET_USED(dev); - RTE_SET_USED(buffers); - RTE_SET_USED(count); - RTE_SET_USED(context); + /* Not FIFO. Just for testing memory read. */ + struct ntb_hw *hw = dev->dev_private; + unsigned int i; + size_t size; + + size = (size_t)context; + for (i = 0; i < count; i++) + rte_memcpy(buffers[i]->buf_addr, hw->mz[i]->addr, size); return 0; } diff --git a/examples/Makefile b/examples/Makefile index 7562424d99..de11dd4875 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -53,6 +53,7 @@ DIRS-y += link_status_interrupt DIRS-$(CONFIG_RTE_LIBRTE_LPM) += load_balancer DIRS-y += multi_process DIRS-y += netmap_compat/bridge +DIRS-y += ntb DIRS-$(CONFIG_RTE_LIBRTE_REORDER) += packet_ordering ifeq ($(CONFIG_RTE_ARCH_X86_64),y) DIRS-y += performance-thread diff --git a/examples/meson.build b/examples/meson.build index 87113bd70b..a046b74ade 100644 --- a/examples/meson.build +++ b/examples/meson.build @@ -30,7 +30,7 @@ all_examples = [ 'multi_process/hotplug_mp', 'multi_process/simple_mp', 'multi_process/symmetric_mp', - 'netmap_compat', 'packet_ordering', + 'netmap_compat', 'ntb', 'packet_ordering', 'performance-thread', 'ptpclient', 'qos_meter', 'qos_sched', 'quota_watermark', 'rxtx_callbacks', diff --git a/examples/ntb/Makefile b/examples/ntb/Makefile new file mode 100644 index 0000000000..5ddd9b95f1 --- /dev/null +++ b/examples/ntb/Makefile @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2019 Intel Corporation + +# binary name +APP = ntb_fwd + +# all source are stored in SRCS-y +SRCS-y := ntb_fwd.c + +# Build using pkg-config variables if possible +$(shell pkg-config --exists libdpdk) +ifeq ($(.SHELLSTATUS),0) + +all: shared +.PHONY: shared static +shared: build/$(APP)-shared + ln -sf $(APP)-shared build/$(APP) +static: build/$(APP)-static + ln -sf $(APP)-static build/$(APP) + +CFLAGS += -D_FILE_OFFSET_BITS=64 +LDFLAGS += -pthread + +PC_FILE := $(shell pkg-config --path libdpdk) +CFLAGS += -O3 $(shell pkg-config --cflags libdpdk) +LDFLAGS_SHARED = $(shell pkg-config --libs libdpdk) +LDFLAGS_STATIC = -Wl,-Bstatic $(shell pkg-config --static --libs libdpdk) + +build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build + $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) + +build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build + $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) + +build: + @mkdir -p $@ + +.PHONY: clean +clean: + rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared + rmdir --ignore-fail-on-non-empty build + +else # Build using legacy build system + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +# Default target, can be overridden by command line or environment +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +include $(RTE_SDK)/mk/rte.vars.mk + +ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y) +$(info This application can only operate in a linuxapp environment, \ +please change the definition of the RTE_TARGET environment variable) +all: +else + +CFLAGS += -D_FILE_OFFSET_BITS=64 +CFLAGS += -O2 +CFLAGS += $(WERROR_FLAGS) +CFLAGS += -DALLOW_EXPERIMENTAL_API + +include $(RTE_SDK)/mk/rte.extapp.mk + +endif +endif diff --git a/examples/ntb/meson.build b/examples/ntb/meson.build new file mode 100644 index 0000000000..9a6288f4f1 --- /dev/null +++ b/examples/ntb/meson.build @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2019 Intel Corporation + +# meson file, for building this example as part of a main DPDK build. +# +# To build this example as a standalone application with an already-installed +# DPDK instance, use 'make' + +if host_machine.system() != 'linux' + build = false +endif +deps += 'rawdev' +cflags += ['-D_FILE_OFFSET_BITS=64'] +sources = files( + 'ntb_fwd.c' +) diff --git a/examples/ntb/ntb_fwd.c b/examples/ntb/ntb_fwd.c new file mode 100644 index 0000000000..c169f01a3d --- /dev/null +++ b/examples/ntb/ntb_fwd.c @@ -0,0 +1,377 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2019 Intel Corporation + */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#define NTB_DRV_NAME_LEN 7 +static uint64_t max_file_size = 0x400000; +static uint8_t interactive = 1; +static uint16_t dev_id; + +/* *** Help command with introduction. *** */ +struct cmd_help_result { + cmdline_fixed_string_t help; +}; + +static void cmd_help_parsed(__attribute__((unused)) void *parsed_result, + struct cmdline *cl, + __attribute__((unused)) void *data) +{ + cmdline_printf( + cl, + "\n" + "The following commands are currently available:\n\n" + "Control:\n" + " quit :" + " Quit the application.\n" + "\nFile transmit:\n" + " send [path] :" + " Send [path] file. (No more than %"PRIu64")\n" + " recv [path] :" + " Receive file to [path]. Make sure sending is done" + " on the other side.\n", + max_file_size + ); + +} + +cmdline_parse_token_string_t cmd_help_help = + TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help"); + +cmdline_parse_inst_t cmd_help = { + .f = cmd_help_parsed, + .data = NULL, + .help_str = "show help", + .tokens = { + (void *)&cmd_help_help, + NULL, + }, +}; + +/* *** QUIT *** */ +struct cmd_quit_result { + cmdline_fixed_string_t quit; +}; + +static void cmd_quit_parsed(__attribute__((unused)) void *parsed_result, + struct cmdline *cl, + __attribute__((unused)) void *data) +{ + /* Stop traffic and Close port. */ + rte_rawdev_stop(dev_id); + rte_rawdev_close(dev_id); + + cmdline_quit(cl); +} + +cmdline_parse_token_string_t cmd_quit_quit = + TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit"); + +cmdline_parse_inst_t cmd_quit = { + .f = cmd_quit_parsed, + .data = NULL, + .help_str = "exit application", + .tokens = { + (void *)&cmd_quit_quit, + NULL, + }, +}; + +/* *** SEND FILE PARAMETERS *** */ +struct cmd_sendfile_result { + cmdline_fixed_string_t send_string; + char filepath[]; +}; + +static void +cmd_sendfile_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_sendfile_result *res = parsed_result; + struct rte_rawdev_buf *pkts_send[1]; + uint64_t rsize, size, link; + uint8_t *buff; + uint32_t val; + FILE *file; + + if (!rte_rawdevs[dev_id].started) { + printf("Device needs to be up first. Try later.\n"); + return; + } + + rte_rawdev_get_attr(dev_id, "link_status", &link); + if (!link) { + printf("Link is not up, cannot send file.\n"); + return; + } + + file = fopen(res->filepath, "r"); + if (file == NULL) { + printf("Fail to open the file.\n"); + return; + } + + fseek(file, 0, SEEK_END); + size = ftell(file); + fseek(file, 0, SEEK_SET); + + /** + * No FIFO now. Only test memory. Limit sending file + * size <= max_file_size. + */ + if (size > max_file_size) { + printf("Warning: The file is too large. Only send first" + " %"PRIu64" bits.\n", max_file_size); + size = max_file_size; + } + + buff = (uint8_t *)malloc(size); + rsize = fread(buff, size, 1, file); + if (rsize != 1) { + printf("Fail to read file.\n"); + fclose(file); + free(buff); + return; + } + + /* Tell remote about the file size. */ + val = size >> 32; + rte_rawdev_set_attr(dev_id, "spad_user_0", val); + val = size; + rte_rawdev_set_attr(dev_id, "spad_user_1", val); + + pkts_send[0] = (struct rte_rawdev_buf *)malloc + (sizeof(struct rte_rawdev_buf)); + pkts_send[0]->buf_addr = buff; + + if (rte_rawdev_enqueue_buffers(dev_id, pkts_send, 1, + (void *)(size_t)size)) { + printf("Fail to enqueue.\n"); + goto clean; + } + printf("Done sending file.\n"); + +clean: + fclose(file); + free(buff); + free(pkts_send[0]); +} + +cmdline_parse_token_string_t cmd_send_file_send = + TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, send_string, + "send"); +cmdline_parse_token_string_t cmd_send_file_filepath = + TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, filepath, NULL); + + +cmdline_parse_inst_t cmd_send_file = { + .f = cmd_sendfile_parsed, + .data = NULL, + .help_str = "send ", + .tokens = { + (void *)&cmd_send_file_send, + (void *)&cmd_send_file_filepath, + NULL, + }, +}; + +/* *** RECEIVE FILE PARAMETERS *** */ +struct cmd_recvfile_result { + cmdline_fixed_string_t recv_string; + char filepath[]; +}; + +static void +cmd_recvfile_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_sendfile_result *res = parsed_result; + struct rte_rawdev_buf *pkts_recv[1]; + uint8_t *buff; + uint64_t val; + size_t size; + FILE *file; + + if (!rte_rawdevs[dev_id].started) { + printf("Device needs to be up first. Try later.\n"); + return; + } + + rte_rawdev_get_attr(dev_id, "link_status", &val); + if (!val) { + printf("Link is not up, cannot receive file.\n"); + return; + } + + file = fopen(res->filepath, "w"); + if (file == NULL) { + printf("Fail to open the file.\n"); + return; + } + + rte_rawdev_get_attr(dev_id, "spad_user_0", &val); + size = val << 32; + rte_rawdev_get_attr(dev_id, "spad_user_1", &val); + size |= val; + + buff = (uint8_t *)malloc(size); + pkts_recv[0] = (struct rte_rawdev_buf *)malloc + (sizeof(struct rte_rawdev_buf)); + pkts_recv[0]->buf_addr = buff; + + if (rte_rawdev_dequeue_buffers(dev_id, pkts_recv, 1, (void *)size)) { + printf("Fail to dequeue.\n"); + goto clean; + } + + fwrite(buff, size, 1, file); + printf("Done receiving to file.\n"); + +clean: + fclose(file); + free(buff); + free(pkts_recv[0]); +} + +cmdline_parse_token_string_t cmd_recv_file_recv = + TOKEN_STRING_INITIALIZER(struct cmd_recvfile_result, recv_string, + "recv"); +cmdline_parse_token_string_t cmd_recv_file_filepath = + TOKEN_STRING_INITIALIZER(struct cmd_recvfile_result, filepath, NULL); + + +cmdline_parse_inst_t cmd_recv_file = { + .f = cmd_recvfile_parsed, + .data = NULL, + .help_str = "recv ", + .tokens = { + (void *)&cmd_recv_file_recv, + (void *)&cmd_recv_file_filepath, + NULL, + }, +}; + +/* list of instructions */ +cmdline_parse_ctx_t main_ctx[] = { + (cmdline_parse_inst_t *)&cmd_help, + (cmdline_parse_inst_t *)&cmd_send_file, + (cmdline_parse_inst_t *)&cmd_recv_file, + (cmdline_parse_inst_t *)&cmd_quit, + NULL, +}; + +/* prompt function, called from main on MASTER lcore */ +static void +prompt(void) +{ + struct cmdline *cl; + + cl = cmdline_stdin_new(main_ctx, "ntb> "); + if (cl == NULL) + return; + + cmdline_interact(cl); + cmdline_stdin_exit(cl); +} + +static void +signal_handler(int signum) +{ + if (signum == SIGINT || signum == SIGTERM) { + printf("\nSignal %d received, preparing to exit...\n", signum); + signal(signum, SIG_DFL); + kill(getpid(), signum); + } +} + +static void +ntb_usage(const char *prgname) +{ + printf("%s [EAL options] -- [options]\n" + "-i : run in interactive mode (default value is 1)\n", + prgname); +} + +static int +parse_args(int argc, char **argv) +{ + char *prgname = argv[0], **argvopt = argv; + int opt, ret; + + /* Only support interactive mode to send/recv file first. */ + while ((opt = getopt(argc, argvopt, "i")) != EOF) { + switch (opt) { + case 'i': + printf("Interactive-mode selected\n"); + interactive = 1; + break; + + default: + ntb_usage(prgname); + return -1; + } + } + + if (optind >= 0) + argv[optind-1] = prgname; + + ret = optind-1; + optind = 1; /* reset getopt lib */ + return ret; +} + +int +main(int argc, char **argv) +{ + int ret, i; + + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); + + ret = rte_eal_init(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Error with EAL initialization.\n"); + + /* Find 1st ntb rawdev. */ + for (i = 0; i < RTE_RAWDEV_MAX_DEVS; i++) + if (rte_rawdevs[i].driver_name && + (strncmp(rte_rawdevs[i].driver_name, "raw_ntb", + NTB_DRV_NAME_LEN) == 0) && (rte_rawdevs[i].attached == 1)) + break; + + if (i == RTE_RAWDEV_MAX_DEVS) + rte_exit(EXIT_FAILURE, "Cannot find any ntb device.\n"); + + dev_id = i; + + argc -= ret; + argv += ret; + + ret = parse_args(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Invalid arguments\n"); + + rte_rawdev_start(dev_id); + + if (interactive) { + sleep(1); + prompt(); + } + + return 0; +} -- 2.20.1