ci: catch coredumps

Parts of the unit tests code rely on forked/secondary processes
(expectedly) failing.
A crash in those situations could be missed so add a check on coredumps
presence after unit tests have run.
When unit tests fail, it can also help checking for coredumps as it
could give more insights on what happened.

In some situations (like explicit call to rte_panic), coredump generation
must be disabled to avoid false positives.

Cc: stable@dpdk.org
Signed-off-by: David Marchand <david.marchand@redhat.com>
Acked-by: Luca Boccassi <bluca@debian.org>
Acked-by: Aaron Conole <aconole@redhat.com>

diff --git a/.ci/linux-build.sh b/.ci/linux-build.sh
index fdbeb5a616042d8ec8f5a1ec29f5bf769f7d9038..91e43a975b8193489a3b46e1f1be5dcb7ca10256 100755 (executable)
--- a/.ci/linux-build.sh
+++ b/.ci/linux-build.sh
@@ -4,7 +4,10 @@ on_error() {
     if [ $? = 0 ]; then
         exit
     fi
-    FILES_TO_PRINT="build/meson-logs/testlog.txt build/.ninja_log build/meson-logs/meson-log.txt"
+    FILES_TO_PRINT="build/meson-logs/testlog.txt"
+    FILES_TO_PRINT="$FILES_TO_PRINT build/.ninja_log"
+    FILES_TO_PRINT="$FILES_TO_PRINT build/meson-logs/meson-log.txt"
+    FILES_TO_PRINT="$FILES_TO_PRINT build/gdb.log"
 
     for pr_file in $FILES_TO_PRINT; do
         if [ -e "$pr_file" ]; then
@@ -30,6 +33,26 @@ install_libabigail() {
     rm ${version}.tar.gz
 }
 
+configure_coredump() {
+    # No point in configuring coredump without gdb
+    which gdb >/dev/null || return 0
+    ulimit -c unlimited
+    sudo sysctl -w kernel.core_pattern=/tmp/dpdk-core.%e.%p
+}
+
+catch_coredump() {
+    ls /tmp/dpdk-core.*.* 2>/dev/null || return 0
+    for core in /tmp/dpdk-core.*.*; do
+        binary=$(sudo readelf -n $core |grep $(pwd)/build/ 2>/dev/null |head -n1)
+        [ -x $binary ] || binary=
+        sudo gdb $binary -c $core \
+            -ex 'info threads' \
+            -ex 'thread apply all bt full' \
+            -ex 'quit'
+    done |tee -a build/gdb.log
+    return 1
+}
+
 if [ "$AARCH64" = "true" ]; then
     # convert the arch specifier
     if [ "$CC_FOR_BUILD" = "gcc" ]; then
@@ -62,7 +85,11 @@ meson build --werror $OPTS
 ninja -C build
 
 if [ "$AARCH64" != "true" ]; then
-    devtools/test-null.sh
+    failed=
+    configure_coredump
+    devtools/test-null.sh || failed="true"
+    catch_coredump
+    [ "$failed" != "true" ]
 fi
 
 if [ "$ABI_CHECKS" = "true" ]; then
@@ -107,5 +134,9 @@ if [ "$ABI_CHECKS" = "true" ]; then
 fi
 
 if [ "$RUN_TESTS" = "true" ]; then
-    sudo meson test -C build --suite fast-tests -t 3
+    failed=
+    configure_coredump
+    sudo meson test -C build --suite fast-tests -t 3 || failed="true"
+    catch_coredump
+    [ "$failed" != "true" ]
 fi

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 2fd1dde86f9b818fc218b5db1c50bc5c31a4e9bc..9175ec43d3acac93650e111b5d7ac990543bbc83 100644 (file)
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -104,6 +104,9 @@ jobs:
       if: env.AARCH64 == 'true'
       run: sudo apt install -y gcc-aarch64-linux-gnu libc6-dev-arm64-cross
         pkg-config-aarch64-linux-gnu
+    - name: Install test tools packages
+      if: env.AARCH64 != 'true' || env.RUN_TESTS == 'true'
+      run: sudo apt install -y gdb
     - name: Install doc generation packages
       if: env.BUILD_DOCS == 'true'
       run: sudo apt install -y doxygen graphviz python3-sphinx
@@ -124,3 +127,4 @@ jobs:
           build/meson-logs/testlog.txt
           build/.ninja_log
           build/meson-logs/meson-log.txt
+          build/gdb.log

diff --git a/app/test/test_debug.c b/app/test/test_debug.c
index 834a7386f554790b60880d1b29690670a410690a..23b24db177394919cd2e52f2d566317e001c9688 100644 (file)
--- a/app/test/test_debug.c
+++ b/app/test/test_debug.c
@@ -4,6 +4,8 @@
 
 #include <stdio.h>
 #include <stdint.h>
+#include <sys/resource.h>
+#include <sys/time.h>
 #include <sys/wait.h>
 #include <unistd.h>
 
@@ -28,9 +30,14 @@ test_panic(void)
 
        pid = fork();
 
-       if (pid == 0)
+       if (pid == 0) {
+               struct rlimit rl;
+
+               /* No need to generate a coredump when panicking. */
+               rl.rlim_cur = rl.rlim_max = 0;
+               setrlimit(RLIMIT_CORE, &rl);
                rte_panic("Test Debug\n");
-       else if (pid < 0){
+       } else if (pid < 0) {
                printf("Fork Failed\n");
                return -1;
        }

diff --git a/app/test/test_mbuf.c b/app/test/test_mbuf.c
index a40f7d48834afebec07b892caa568413307ef40b..47a7b197d78c338c950296098b109a218c95a217 100644 (file)
--- a/app/test/test_mbuf.c
+++ b/app/test/test_mbuf.c
@@ -1174,6 +1174,8 @@ err:
 }
 
 #include <unistd.h>
+#include <sys/resource.h>
+#include <sys/time.h>
 #include <sys/wait.h>
 
 /* use fork() to test mbuf errors panic */
@@ -1186,9 +1188,14 @@ verify_mbuf_check_panics(struct rte_mbuf *buf)
        pid = fork();
 
        if (pid == 0) {
+               struct rlimit rl;
+
+               /* No need to generate a coredump when panicking. */
+               rl.rlim_cur = rl.rlim_max = 0;
+               setrlimit(RLIMIT_CORE, &rl);
                rte_mbuf_sanity_check(buf, 1); /* should panic */
                exit(0);  /* return normally if it doesn't panic */
-       } else if (pid < 0){
+       } else if (pid < 0) {
                printf("Fork Failed\n");
                return -1;
        }