From 3cd7290c1603db0606264c2f41f06204fde03402 Mon Sep 17 00:00:00 2001 From: David Marchand Date: Mon, 25 Jan 2021 16:05:39 +0100 Subject: [PATCH] ci: catch coredumps Parts of the unit tests code rely on forked/secondary processes (expectedly) failing. A crash in those situations could be missed so add a check on coredumps presence after unit tests have run. When unit tests fail, it can also help checking for coredumps as it could give more insights on what happened. In some situations (like explicit call to rte_panic), coredump generation must be disabled to avoid false positives. Cc: stable@dpdk.org Signed-off-by: David Marchand Acked-by: Luca Boccassi Acked-by: Aaron Conole --- .ci/linux-build.sh | 37 ++++++++++++++++++++++++++++++++++--- .github/workflows/build.yml | 4 ++++ app/test/test_debug.c | 11 +++++++++-- app/test/test_mbuf.c | 9 ++++++++- 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/.ci/linux-build.sh b/.ci/linux-build.sh index fdbeb5a616..91e43a975b 100755 --- a/.ci/linux-build.sh +++ b/.ci/linux-build.sh @@ -4,7 +4,10 @@ on_error() { if [ $? = 0 ]; then exit fi - FILES_TO_PRINT="build/meson-logs/testlog.txt build/.ninja_log build/meson-logs/meson-log.txt" + FILES_TO_PRINT="build/meson-logs/testlog.txt" + FILES_TO_PRINT="$FILES_TO_PRINT build/.ninja_log" + FILES_TO_PRINT="$FILES_TO_PRINT build/meson-logs/meson-log.txt" + FILES_TO_PRINT="$FILES_TO_PRINT build/gdb.log" for pr_file in $FILES_TO_PRINT; do if [ -e "$pr_file" ]; then @@ -30,6 +33,26 @@ install_libabigail() { rm ${version}.tar.gz } +configure_coredump() { + # No point in configuring coredump without gdb + which gdb >/dev/null || return 0 + ulimit -c unlimited + sudo sysctl -w kernel.core_pattern=/tmp/dpdk-core.%e.%p +} + +catch_coredump() { + ls /tmp/dpdk-core.*.* 2>/dev/null || return 0 + for core in /tmp/dpdk-core.*.*; do + binary=$(sudo readelf -n $core |grep $(pwd)/build/ 2>/dev/null |head -n1) + [ -x $binary ] || binary= + sudo gdb $binary -c $core \ + -ex 'info threads' \ + -ex 'thread apply all bt full' \ + -ex 'quit' + done |tee -a build/gdb.log + return 1 +} + if [ "$AARCH64" = "true" ]; then # convert the arch specifier if [ "$CC_FOR_BUILD" = "gcc" ]; then @@ -62,7 +85,11 @@ meson build --werror $OPTS ninja -C build if [ "$AARCH64" != "true" ]; then - devtools/test-null.sh + failed= + configure_coredump + devtools/test-null.sh || failed="true" + catch_coredump + [ "$failed" != "true" ] fi if [ "$ABI_CHECKS" = "true" ]; then @@ -107,5 +134,9 @@ if [ "$ABI_CHECKS" = "true" ]; then fi if [ "$RUN_TESTS" = "true" ]; then - sudo meson test -C build --suite fast-tests -t 3 + failed= + configure_coredump + sudo meson test -C build --suite fast-tests -t 3 || failed="true" + catch_coredump + [ "$failed" != "true" ] fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2fd1dde86f..9175ec43d3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -104,6 +104,9 @@ jobs: if: env.AARCH64 == 'true' run: sudo apt install -y gcc-aarch64-linux-gnu libc6-dev-arm64-cross pkg-config-aarch64-linux-gnu + - name: Install test tools packages + if: env.AARCH64 != 'true' || env.RUN_TESTS == 'true' + run: sudo apt install -y gdb - name: Install doc generation packages if: env.BUILD_DOCS == 'true' run: sudo apt install -y doxygen graphviz python3-sphinx @@ -124,3 +127,4 @@ jobs: build/meson-logs/testlog.txt build/.ninja_log build/meson-logs/meson-log.txt + build/gdb.log diff --git a/app/test/test_debug.c b/app/test/test_debug.c index 834a7386f5..23b24db177 100644 --- a/app/test/test_debug.c +++ b/app/test/test_debug.c @@ -4,6 +4,8 @@ #include #include +#include +#include #include #include @@ -28,9 +30,14 @@ test_panic(void) pid = fork(); - if (pid == 0) + if (pid == 0) { + struct rlimit rl; + + /* No need to generate a coredump when panicking. */ + rl.rlim_cur = rl.rlim_max = 0; + setrlimit(RLIMIT_CORE, &rl); rte_panic("Test Debug\n"); - else if (pid < 0){ + } else if (pid < 0) { printf("Fork Failed\n"); return -1; } diff --git a/app/test/test_mbuf.c b/app/test/test_mbuf.c index a40f7d4883..47a7b197d7 100644 --- a/app/test/test_mbuf.c +++ b/app/test/test_mbuf.c @@ -1174,6 +1174,8 @@ err: } #include +#include +#include #include /* use fork() to test mbuf errors panic */ @@ -1186,9 +1188,14 @@ verify_mbuf_check_panics(struct rte_mbuf *buf) pid = fork(); if (pid == 0) { + struct rlimit rl; + + /* No need to generate a coredump when panicking. */ + rl.rlim_cur = rl.rlim_max = 0; + setrlimit(RLIMIT_CORE, &rl); rte_mbuf_sanity_check(buf, 1); /* should panic */ exit(0); /* return normally if it doesn't panic */ - } else if (pid < 0){ + } else if (pid < 0) { printf("Fork Failed\n"); return -1; } -- 2.20.1