diff --git a/Makefile b/Makefile index b220afe9fd79049fea04e0634ace45f20d3ec1c7..e121d3c87206ce7b5a87b8cf4163df229b1eceaf 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,7 @@ GLUON_TMPDIR ?= tmp GLUON_OUTPUTDIR ?= output GLUON_IMAGEDIR ?= $(GLUON_OUTPUTDIR)/images GLUON_PACKAGEDIR ?= $(GLUON_OUTPUTDIR)/packages +GLUON_DEBUGDIR ?= $(GLUON_OUTPUTDIR)/debug GLUON_TARGETSDIR ?= targets GLUON_PATCHESDIR ?= patches @@ -56,7 +57,7 @@ endef GLUON_VARS = \ GLUON_RELEASE GLUON_REGION GLUON_MULTIDOMAIN GLUON_AUTOREMOVE GLUON_DEBUG GLUON_MINIFY GLUON_DEPRECATED \ - GLUON_DEVICES GLUON_TARGETSDIR GLUON_PATCHESDIR GLUON_TMPDIR GLUON_IMAGEDIR GLUON_PACKAGEDIR \ + GLUON_DEVICES GLUON_TARGETSDIR GLUON_PATCHESDIR GLUON_TMPDIR GLUON_IMAGEDIR GLUON_PACKAGEDIR GLUON_DEBUGDIR \ GLUON_SITEDIR GLUON_RELEASE GLUON_BRANCH GLUON_LANGS GLUON_BASE_FEEDS \ GLUON_TARGET BOARD SUBTARGET diff --git a/docs/dev/debugging.rst b/docs/dev/debugging.rst new file mode 100644 index 0000000000000000000000000000000000000000..413f1c975aa9799a5376f45bd46b75303d84fd4a --- /dev/null +++ b/docs/dev/debugging.rst @@ -0,0 +1,51 @@ +Debugging +========= + + +.. _dev-debugging-kernel-oops: + +Kernel Oops +----------- + +Sometimes a running Linux kernel detects an error during runtime that canot +be corrected. +This usually generates a stack trace that points to the location in the code +that caused the oops. + +Linux kernels in Gluon (and OpenWrt) are stripped. +That means they do not contain any debug symbols. +On one hand this leads to a smaller binary and faster loading times on the +target. +On the other hand this means that in a case of a stack trace the unwinder +can only print memory locations and no further debugging information. + +Gluon stores a compressed kernel with debug symbols for every target +in the directory `output/debug/`. +These kernels should be kept along with the images as long as the images +are in use. +This allows the developer to analyse a stack trace later. + +Decoding Stacktraces +.................... + +The tooling is contained in the kernel source tree in the file +`decode_stacktrace.sh <https://github.com/torvalds/linux/blob/master/scripts/decode_stacktrace.sh>`__. +This file and the needed source tree are available in the directory: :: + + openwrt/build_dir/target-<architecture>/linux-<architecture>/linux-<version>/ + +.. note:: + Make sure to use a kernel tree that matches the version and patches + that was used to build the kernel. + If in doubt just re-build the images for the target. + +Some more information on how to use this tool can be found at +`LWN <https://lwn.net/Articles/592724/>`__. + +Obtaining Stacktraces +..................... + +On many targets stacktraces can be read from the following +location after reboot: :: + + /sys/kernel/debug/crashlog diff --git a/docs/index.rst b/docs/index.rst index 9abd8dc580e06f929a538cee9a2577356cc62288..cf767073f2dcb4b173983709bd8b2d46f7a4e25f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -43,6 +43,7 @@ Several Freifunk communities in Germany use Gluon as the foundation of their Fre dev/mac_addresses dev/site_library dev/build + dev/debugging .. toctree:: :caption: gluon-web Reference diff --git a/docs/user/getting_started.rst b/docs/user/getting_started.rst index db642bafc4c544a4a8ccf3be89ba70de266a600e..36d9260e8e80b2f2784651b4a2f041d8b065d505 100644 --- a/docs/user/getting_started.rst +++ b/docs/user/getting_started.rst @@ -113,6 +113,12 @@ symlinks, you can use the following command to resolve these links while copying cp -rL output/images /var/www +The directory `output/debug` contains a compressed kernel image for each +architecture. +These can be used for debugging and should be stored along with the images to +allow debugging of kernel problems on devices in the field. +See :ref:`Debugging <dev-debugging-kernel-oops>` for more information. + Cleaning the build tree ....................... diff --git a/scripts/copy_output.lua b/scripts/copy_output.lua index 91547964fb5a13c62a29ad7cc6eef71ac17e5236..9f14fba5049628cb2a87928b0a2e321f5a30f4fa 100755 --- a/scripts/copy_output.lua +++ b/scripts/copy_output.lua @@ -27,6 +27,7 @@ end mkdir(env.GLUON_IMAGEDIR..'/factory') mkdir(env.GLUON_IMAGEDIR..'/sysupgrade') mkdir(env.GLUON_IMAGEDIR..'/other') +mkdir(env.GLUON_DEBUGDIR) lib.include(target) @@ -66,6 +67,20 @@ for _, images in pairs(lib.images) do end end +-- copy kernel image with debug symbols +local kernel_debug_glob = string.format('%s/gluon-\0-%s-kernel-debug.tar.zst', + env.GLUON_DEBUGDIR, + target) +lib.exec {'rm', '-f', kernel_debug_glob} +local kernel_debug_source = string.format('openwrt/bin/targets/%s/kernel-debug.tar.zst', + bindir) +local kernel_debug_dest = string.format('%s/gluon-%s-%s-%s-kernel-debug.tar.zst', + env.GLUON_DEBUGDIR, + lib.site_code, + env.GLUON_RELEASE, + target) +lib.exec {'cp', kernel_debug_source, kernel_debug_dest} + -- Copy opkg repo if lib.opkg and (env.GLUON_DEVICES or '') == '' then diff --git a/targets/generic b/targets/generic index e02858210421b81eb1cb68dc6ce17c473f3ccc74..c7f284523ecda3f671982d4178ac85a04a3ca49e 100644 --- a/targets/generic +++ b/targets/generic @@ -47,6 +47,8 @@ try_config('TARGET_SQUASHFS_BLOCK_SIZE', 256) config('KERNEL_IP_MROUTE', false) config('KERNEL_IPV6_MROUTE', false) +config('COLLECT_KERNEL_DEBUG', true) + try_config('TARGET_MULTI_PROFILE', true) try_config('TARGET_PER_DEVICE_ROOTFS', true)