contrib/lsupgrade.sh

-#!/bin/bash
+#!/usr/bin/env bash
 
+set -e
 # Script to list all upgrade scripts in a clear manner
 # Limitations:
 #  * Does only show scripts of packages whose `files'/`luasrc' directories represent the whole image filesystem (which are all Gluon packages)
@@ -27,7 +28,7 @@ fi
 
 pushd "$(dirname "$0")/.." >/dev/null
 
-find ./package packages -name Makefile | while read makefile; do
+find ./package packages -name Makefile | grep -v '^packages/packages/' | while read -r makefile; do
 	dir="$(dirname "$makefile")"
 
 	pushd "$dir" >/dev/null
@@ -36,13 +37,12 @@ find ./package packages -name Makefile | while read makefile; do
 	dirname="$(dirname "$dir" | cut -d/ -f 3-)"
 	package="$(basename "$dir")"
 
-	for file in "${SUFFIX1}"/*; do
-		echo "${GREEN}$(basename "${file}")${RESET}" "(${BLUE}${repo}${RESET}/${dirname}${dirname:+/}${RED}${package}${RESET}/${SUFFIX1})"
-	done
-	for file in "${SUFFIX2}"/*; do
-		echo "${GREEN}$(basename "${file}")${RESET}" "(${BLUE}${repo}${RESET}/${dirname}${dirname:+/}${RED}${package}${RESET}/${SUFFIX2})"
+	for file in "${SUFFIX1}"/* "${SUFFIX2}"/*; do
+		basename="$(basename "${file}")"
+		suffix="$(dirname "${file}")"
+		printf "%s\t%s\n" "${basename}" "${BLUE}${repo}${RESET}/${dirname}${dirname:+/}${RED}${package}${RESET}/${suffix}/${GREEN}${basename}${RESET}"
 	done
 	popd >/dev/null
-done | sort
+done | sort | cut -f2-
 
 popd >/dev/null

contrib/push_pkg.sh

+#!/bin/sh
+
+set -e
+
+topdir="$(realpath "$(dirname "${0}")/../openwrt")"
+
+# defaults to qemu run script
+ssh_host=localhost
+build_only=0
+preserve_config=1
+
+print_help() {
+	echo "$0 [OPTIONS] PACKAGE_DIR [PACKAGE_DIR] ..."
+	echo ""
+	echo " -h          print this help"
+	echo " -r HOST     use a remote machine as target machine. By default if this"
+	echo "             option is not given, push_pkg.sh will use a locally"
+	echo "             running qemu instance started by run_qemu.sh."
+	echo " -p PORT     use PORT as ssh port (default is 22)"
+	echo " -b          build only, do not push"
+	echo " -P          do not preserve /etc/config. By default, if a package"
+	echo "             defines a config file in /etc/config, this config file"
+	echo "             will be preserved. If you specify this flag, the package"
+	echo "             default will be installed instead."
+	echo ""
+	echo ' To change gluon variables, run e.g. "make config GLUON_MINIFY=0"'
+	echo ' because then the gluon logic will be triggered, and openwrt/.config'
+	echo ' will be regenerated. The variables from openwrt/.config are already'
+	echo ' automatically used for this script.'
+	echo
+}
+
+while getopts "p:r:hbP" opt
+do
+	case $opt in
+		P) preserve_config=0;;
+		p) ssh_port="${OPTARG}";;
+		r) ssh_host="${OPTARG}"; [ -z "$ssh_port" ] && ssh_port=22;;
+		b) build_only=1;;
+		h) print_help; exit 0;;
+		*) ;;
+	esac
+done
+shift $(( OPTIND - 1 ))
+
+[ -z "$ssh_port" ] && ssh_port=2223
+
+if [ "$build_only" -eq 0 ]; then
+	remote_info=$(ssh -p "${ssh_port}" "root@${ssh_host}" '
+		source /etc/os-release
+		printf "%s\\t%s\\n" "$OPENWRT_BOARD" "$OPENWRT_ARCH"
+	')
+	REMOTE_OPENWRT_BOARD="$(echo "$remote_info" | cut -f 1)"
+	REMOTE_OPENWRT_ARCH="$(echo "$remote_info" | cut -f 2)"
+
+	# check target
+	if ! grep -q "CONFIG_TARGET_ARCH_PACKAGES=\"${REMOTE_OPENWRT_ARCH}\"" "${topdir}/.config"; then
+		echo "Configured OpenWrt Target is not matching with the target machine!" 1>&2
+		echo
+		printf "%s" "    Configured architecture: " 1>&2
+		grep "CONFIG_TARGET_ARCH_PACKAGES" "${topdir}/.config" 1>&2
+		echo "Target machine architecture: ${REMOTE_OPENWRT_ARCH}" 1>&2
+		echo 1>&2
+		echo "To switch the local with the run with the corresponding GLUON_TARGET:"  1>&2
+		echo "  make GLUON_TARGET=... config" 1>&2
+		exit 1
+	fi
+fi
+
+if [ $# -lt 1 ]; then
+	echo ERROR: Please specify a PACKAGE_DIR. For example:
+	echo
+	echo " \$ $0 package/gluon-core"
+	exit 1
+fi
+
+while [ $# -gt 0 ]; do
+
+	pkgdir="$1"; shift
+	echo "Package: ${pkgdir}"
+
+	if ! [ -f "${pkgdir}/Makefile" ]; then
+		echo "ERROR: ${pkgdir} does not contain a Makefile"
+		exit 1
+	fi
+
+	if ! grep -q BuildPackage "${pkgdir}/Makefile"; then
+		echo "ERROR: ${pkgdir}/Makefile does not contain a BuildPackage command"
+		exit 1
+	fi
+
+	opkg_packages="$(make TOPDIR="${topdir}" -C "${pkgdir}" DUMP=1 | awk '/^Package: / { print $2 }')"
+
+	search_package() {
+		find "$2" -name "$1_*.ipk" -printf '%f\n'
+	}
+
+	make TOPDIR="${topdir}" -C "${pkgdir}" clean
+	make TOPDIR="${topdir}" -C "${pkgdir}" compile
+
+	if [ "$build_only" -eq 1 ]; then
+		continue
+	fi
+
+	# IPv6 addresses need brackets around the ${ssh_host} for scp!
+	if echo "${ssh_host}" | grep -q :; then
+		BL=[
+		BR=]
+	fi
+
+	for pkg in ${opkg_packages}; do
+
+		for feed in "${topdir}/bin/packages/${REMOTE_OPENWRT_ARCH}/"*/ "${topdir}/bin/targets/${REMOTE_OPENWRT_BOARD}/packages/"; do
+			printf "%s" "searching ${pkg} in ${feed}: "
+			filename=$(search_package "${pkg}" "${feed}")
+			if [ -n "${filename}" ]; then
+				echo found!
+				break
+			else
+				echo not found
+			fi
+		done
+
+		if [ "$preserve_config" -eq 0 ]; then
+			opkg_flags=" --force-maintainer"
+		fi
+
+		# shellcheck disable=SC2029
+		if [ -n "$filename" ]; then
+			scp -O -P "${ssh_port}" "$feed/$filename" "root@${BL}${ssh_host}${BR}:/tmp/${filename}"
+			ssh -p "${ssh_port}" "root@${ssh_host}" "
+				set -e
+				echo Running opkg:
+				opkg install --force-reinstall ${opkg_flags} '/tmp/${filename}'
+				rm '/tmp/${filename}'
+				gluon-reconfigure
+			"
+		else
+			# Some packages (e.g. procd-seccomp) seem to contain BuildPackage commands
+			# which do not generate *.ipk files. Till this point, I am not aware why
+			# this is happening. However, dropping a warning if the corresponding
+			# *.ipk is not found (maybe due to other reasons as well), seems to
+			# be more reasonable than aborting. Before this commit, the command
+			# has failed.
+			echo "Warning: ${pkg}*.ipk not found! Ignoring." 1>&2
+		fi
+
+	done
+done

contrib/run_qemu.sh

+#!/bin/sh
+
+# Note: You can exit the qemu instance by first pressing "CTRL + a" then "c".
+#       Then you enter the command mode of qemu and can exit by typing "quit".
+
+qemu-system-x86_64 \
+	-d 'cpu_reset' \
+	-enable-kvm \
+	-gdb tcp::1234 \
+	-nographic \
+	-netdev user,id=wan,hostfwd=tcp::2223-10.0.2.15:22 \
+	-device virtio-net-pci,netdev=wan,addr=0x06,id=nic1 \
+	-netdev user,id=lan,hostfwd=tcp::6080-192.168.1.1:80,hostfwd=tcp::2222-192.168.1.1:22,net=192.168.1.100/24 \
+	-device virtio-net-pci,netdev=lan,addr=0x05,id=nic2 \
+	"$@"

contrib/sign.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-if [ $# -ne 2 -o "-h" = "$1" -o "--help" = "$1" -o ! -r "$1" -o ! -r "$2" ]; then
+if [ $# -ne 2 ] || [ "-h" = "$1" ] || [ "--help" = "$1" ] || [ ! -r "$1" ] || [ ! -r "$2" ]; then
 	cat <<EOHELP
 Usage: $0 <secret> <manifest>
 
@@ -29,11 +29,22 @@ lower="$(mktemp)"
 
 trap 'rm -f "$upper" "$lower"' EXIT
 
-awk 'BEGIN    { sep=0 }
-     /^---$/ { sep=1; next }
-              { if(sep==0) print > "'"$upper"'";
-                else       print > "'"$lower"'"}' \
-    "$manifest"
+awk 'BEGIN    {
+	sep = 0
+}
+
+/^---$/ {
+	sep = 1;
+	next
+}
+
+{
+	if(sep == 0) {
+		print > "'"$upper"'"
+	} else {
+		print > "'"$lower"'"
+	}
+}' "$manifest"
 
 ecdsasign "$upper" < "$SECRET" >> "$lower"
 

contrib/sigtest.sh

 #!/bin/sh
 
-if [ $# -eq 0 -o "-h" = "$1" -o "-help" = "$1" -o "--help" = "$1" ]; then
+if [ $# -eq 0 ] || [ "-h" = "$1" ] || [ "-help" = "$1" ] || [ "--help" = "$1" ]; then
 	cat <<EOHELP
 Usage: $0 <public> <signed manifest>
 
@@ -21,13 +21,24 @@ upper="$(mktemp)"
 lower="$(mktemp)"
 ret=1
 
-awk "BEGIN    { sep=0 }
-    /^---\$/ { sep=1; next }
-              { if(sep==0) print > \"$upper\";
-                else       print > \"$lower\"}" \
-    "$manifest"
+awk 'BEGIN    {
+	sep = 0
+}
 
-while read line
+/^---$/ {
+	sep = 1;
+	next
+}
+
+{
+	if(sep == 0) {
+		print > "'"$upper"'"
+	} else {
+		print > "'"$lower"'"
+	}
+}' "$manifest"
+
+while read -r line
 do
 	if ecdsaverify -s "$line" -p "$public" "$upper"; then
 		ret=0

docs/_static/css/custom.css

+.strike {
+	text-decoration: line-through;
+}

docs/conf.py

@@ -20,11 +20,11 @@
 # -- Project information -----------------------------------------------------
 
 project = 'Gluon'
-copyright = '2015-2019, Project Gluon'
+copyright = 'Project Gluon'
 author = 'Project Gluon'
 
 # The short X.Y version
-version = '2018.2+'
+version = '2023.2.4'
 # The full version, including alpha/beta/rc tags
 release = version
 
@@ -48,7 +48,7 @@ templates_path = ['_templates']
 # You can specify multiple suffix as a list of string:
 #
 # source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = {'.rst': 'restructuredtext'}
 
 # The master toctree document.
 master_doc = 'index'
@@ -58,7 +58,7 @@ master_doc = 'index'
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -71,6 +71,13 @@ pygments_style = None
 # Don't highlight code blocks unless requested explicitly
 highlight_language = 'none'
 
+# Ignore links to the config mode, as well as anchors on on hackint, which are
+# used to mark channel names and do not exist. Regular links are not effected.
+linkcheck_ignore = [
+    'http://192.168.1.1',
+    'https://chat.hackint.org'
+]
+
 
 # -- Options for HTML output -------------------------------------------------
 
@@ -89,7 +96,7 @@ html_theme = 'sphinx_rtd_theme'
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 #
-# html_static_path = ['_static']
+html_static_path = ['_static']
 
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.
@@ -101,6 +108,10 @@ html_theme = 'sphinx_rtd_theme'
 #
 # html_sidebars = {}
 
+# These paths are either relative to html_static_path
+# or fully qualified paths (eg. https://...)
+html_css_files = ['css/custom.css']
+
 
 # -- Options for HTMLHelp output ---------------------------------------------
 

docs/dev/basics.rst

@@ -21,8 +21,9 @@ webbrowser. You're welcome to join us!
 
 .. _#gluon: ircs://irc.hackint.org/#gluon
 .. _hackint: https://hackint.org/
-.. _webchat: https://webirc.hackint.org/#irc://irc.hackint.org/#gluon
+.. _webchat: https://chat.hackint.org/?join=gluon
 
+.. _working-with-repositories:
 
 Working with repositories
 -------------------------
@@ -52,6 +53,14 @@ and you can try rebasing it onto the new `base` branch yourself and after that c
 Always call `make update-patches` after making changes to a module repository as `make update` will overwrite your
 commits, making `git reflog` the only way to recover them!
 
+::
+
+  make refresh-patches
+
+In order to refresh patches when updating feeds or the OpenWrt base, `make refresh-patches` applies and updates all of their patches without installing feed packages to the OpenWrt build system.
+
+This command speeds up the maintenance of updating OpenWrt and feeds.
+
 Development Guidelines
 ----------------------
 Lua should be used instead of sh whenever sensible. The following criteria
@@ -66,6 +75,11 @@ the code in the project is formatted in the same way. The following basic rules
 apply:
 
 - use tabs instead of spaces
-- trailing whitespaces must be eliminated
+- trailing whitespace characters must be eliminated
+- files need to end with a final newline
+- newlines need to have Unix line endings (lf)
+
+To that end we provide a ``.editorconfig`` configuration, which is supported by most
+of the editors out there.
 
 If you add Lua scripts to gluon, check formatting with ``luacheck``.

docs/dev/build.rst

+Build system
+============
+
+This page explains internals of the Gluon build system. It is currently very
+incomplete; please contribute if you can!
+
+Feed management
+---------------
+
+Rather that relying on the *feed.conf* mechanism of OpenWrt directly, Gluon
+manages its feeds (*"modules"*) using a collection of scripts. This solution was
+selected for multiple reasons:
+
+- Feeds lists from Gluon base and the site repository are combined
+- Patchsets are applied to downloaded feed repositories automatically
+
+The following variables specifically affect the feed management:
+
+GLUON_FEEDS
+    List of base feeds; defined in file *modules* in Gluon base
+
+GLUON_SITE_FEED
+    List of site feeds; defined in file *modules* in site config
+
+\*_REPO, \*_BRANCH, \*_COMMIT
+    Git repository URL, branch and
+    commit ID of the feeds to use. The branch name may be omitted; the default
+    branch will be used in this case.
+
+GLUON_BASE_FEEDS
+    Additional feed definitions to be added to *feeds.conf*
+    verbatim. By default, this contains a reference to the Gluon base packages;
+    when using the Gluon build system to build a non-Gluon system, the variable
+    can be set to the empty string.
+
+Helper scripts
+--------------
+
+Several tasks of the build process have been separated from the Makefile into
+external scripts, which are stored in the *scripts* directory. This was done to
+ease maintenance of these scripts and the Makefile, by avoiding a lot of escaping.
+These scripts are either bash or Lua scripts that run on the build system.
+
+default_feeds.sh
+    Defines the constant ``DEFAULT_FEEDS`` with the names of all feeds listed in
+    *openwrt/feeds.conf.default*. This script is only used as an include by other
+    scripts.
+
+feeds.sh
+    Creates the *openwrt/feeds.conf* file from ``FEEDS`` and ``DEFAULT_FEEDS``. The
+    feeds from ``FEEDS`` are linked to the matching subfolder of *packages/* and not
+    explicitly defined feeds of ``DEFAULT_FEEDS`` are setup as dummy (src-dummy).
+    This *openwrt/feeds.conf* is used to reinstall all packages of all feeds with
+    the *openwrt/scripts/feeds* tool.
+
+modules.sh
+    Defines the constants ``GLUON_MODULES`` and ``FEEDS`` by reading the *modules*
+    files of the Gluon repository root and the site configuration. The returned
+    variables look like:
+
+    - ``FEEDS``: "*feedA feedB ...*"
+    - ``GLUON_MODULES``: "*openwrt packages/feedA packages/feedB ...*"
+
+    This script is only used as an include by other scripts.
+
+patch.sh
+    (Re-)applies the patches from the *patches* directory to all ``GLUON_MODULES``
+    and checks out the files to the filesystem.
+    This is done for each repo by:
+
+    - creating a temporary clone of the repo to patch
+      - only branch *base* is used
+    - applying all patches via *git am* on top of this temporary *base* branch
+      - this branch is named *patched*
+    - copying the temporary clone to the *openwrt* (for OpenWrt Base) or
+      *packages* (for feeds) folder
+      - *git fetch* is used with the temporary clone as source
+      - *git checkout* is called to update the filesystem
+    - updating all git submodules
+
+    This solution with a temporary clone ensures that the timestamps of checked
+    out files are not changed by any intermediate patch steps, but only when
+    updating the checkout with the final result. This avoids triggering unnecessary
+    rebuilds.
+
+update.sh
+    Sets up a working clone of the ``GLUON_MODULES`` (external repos) from the external
+    source and installs it into *packages/* directory. It simply tries to set the *base*
+    branch of the cloned repo to the correct commit. If this fails it fetches the
+    upstream branch and tries again to set the local *base* branch.
+
+getversion.sh
+    Used to determine the version numbers of the repositories of Gluon and the
+    site configuration, to be included in the built firmware images as
+    */lib/gluon/gluon-version* and */lib/gluon/site-version*.
+
+    By default, this uses ``git describe`` to generate a version number based
+    on the last git tag. This can be overridden by putting a file called
+    *.scmversion* into the root of the respective repositories.
+
+    A command like ``rm -f .scmversion; echo "$(./scripts/getversion.sh .)" > .scmversion``
+    can be used before applying local patches to ensure that the reported
+    version numbers refer to an upstream commit ID rather than an arbitrary
+    local one after ``git am``.

docs/dev/debugging.rst

+Debugging
+=========
+
+
+.. _dev-debugging-kernel-oops:
+
+Kernel Oops
+-----------
+
+Sometimes a running Linux kernel detects an error during runtime that can't
+be corrected.
+This usually generates a stack trace that points to the location in the code
+that caused the oops.
+
+Linux kernels in Gluon (and OpenWrt) are stripped.
+That means they do not contain any debug symbols.
+On one hand this leads to a smaller binary and faster loading times on the
+target.
+On the other hand this means that in a case of a stack trace the unwinder
+can only print memory locations and no further debugging information.
+
+Gluon stores a compressed kernel with debug symbols for every target
+in the directory `output/debug/`.
+These kernels should be kept along with the images as long as the images
+are in use.
+This allows the developer to analyse a stack trace later.
+
+Decoding Stacktraces
+....................
+
+The tooling is contained in the kernel source tree in the file
+`decode_stacktrace.sh <https://github.com/torvalds/linux/blob/master/scripts/decode_stacktrace.sh>`__.
+This file and the needed source tree are available in the directory: ::
+
+  openwrt/build_dir/target-<architecture>/linux-<architecture>/linux-<version>/
+
+.. note::
+  Make sure to use a kernel tree that matches the version and patches
+  that was used to build the kernel.
+  If in doubt just re-build the images for the target.
+
+Some more information on how to use this tool can be found at
+`LWN <https://lwn.net/Articles/592724/>`__.
+
+Obtaining Stacktraces
+.....................
+
+On many targets stack traces can be read from the following
+location after reboot: ::
+
+  /sys/kernel/debug/crashlog

docs/dev/hardware.rst

-Adding support for new hardware
-===============================
+Adding hardware support
+=======================
 This page will give a short overview on how to add support
 for new hardware to Gluon.
 
 Hardware requirements
 ---------------------
-Having an ath9k (or ath10k) based WLAN adapter is highly recommended,
+Having an ath9k, ath10k or mt76 based WLAN adapter is highly recommended,
 although other chipsets may also work. VAP (multiple SSID) support
-is a requirement.
+with simultaneous AP + Mesh Point (802.11s) operation is required.
 
+Device checklist
+----------------
+The description of pull requests adding device support must include the
+`device integration checklist
+<https://github.com/freifunk-gluon/gluon/wiki/Device-Integration-checklist>`_.
+The checklist ensures that core functionality of Gluon is well supported on the
+device.
 
-.. _hardware-adding-profiles:
+.. _device-class-definition:
 
-Adding profiles
----------------
-The vast majority of devices with ath9k WLAN is based on the ar71xx target of OpenWrt.
-If the hardware you want to add support for is ar71xx, adding a new profile
-is sufficient.
+Device classes
+--------------
+All supported hardware is categorized into "device classes". This allows to
+adjust the feature set of Gluon to the different hardware's capabilities via
+``site.mk`` without having to list individual devices.
 
-Profiles are defined in ``targets/*`` in a shell-based DSL (so common shell
-command syntax like ``if`` can be used).
+There are currently two devices classes defined: "standard" and "tiny". The
+"tiny" class contains all devices that do not meet the following requirements:
 
-The ``device`` command is used to define an image build for a device. It takes
-two or three parameters.
+- At least 7 MiB of usable firmware space
+- At least 64 MiB of RAM (128MiB for devices with ath10k radio)
 
-The first parameter defines the Gluon profile name, which is used to refer to the
-device and is part of the generated image name. The profile name must be same as
-the output of the following command (on the target device), so the autoupdater
-can work::
+Target configuration
+--------------------
+Gluon's hardware support is based on OpenWrt's. For each supported target,
+a configuration file exists at ``targets/<target>-<subtarget>`` (or just
+``target/<target>`` for targets without subtargets) that contains all
+Gluon-specific settings for the target. The generic configuration
+``targets/generic`` contains settings that affect all targets.
 
-    lua -e 'print(require("platform_info").get_image_name())'
+All targets must be listed in ``target/targets.mk``.
 
-While porting Gluon to a new device, it might happen that the profile name is
-unknown. Best practise is to generate an image first by using an arbitrary value
-and then executing the lua command on the device and use its output from then on.
+The target configuration language is based on Lua, so Lua's syntax for variables
+and control structures can be used.
 
-The second parameter defines the name of the image files generated by OpenWrt. Usually,
-it is also the OpenWrt profile name; for devices that still use the old image build
-code, a third parameter with the OpenWrt profile name can be passed. The profile names
-can be found in the image Makefiles in ``openwrt/target/linux/<target>/image/Makefile``.
+Device definitions
+~~~~~~~~~~~~~~~~~~
+To configure a device to be built for Gluon, the ``device`` function is used.
+In the simplest case, only two arguments are passed, for example:
 
-Examples::
+.. code-block:: lua
 
-    device tp-link-tl-wr1043n-nd-v1 tl-wr1043nd-v1
-    device alfa-network-hornet-ub hornet-ub HORNETUB
+  device('tp-link-tl-wdr3600-v1', 'tplink_tl-wdr3600-v1')
 
-Suffixes and extensions
-'''''''''''''''''''''''
-
-By default, image files are expected to have the extension ``.bin``. In addition,
-the images generated by OpenWrt have a suffix before the extension that defaults to
-``-squashfs-factory`` and ``-squashfs-sysupgrade``.
-
-This can be changed using the ``factory`` and ``sysupgrade`` commands, either at
-the top of the file to set the defaults for all images, or for a single image. There
-are three forms with 0 to 2 arguments (all work with ``sysupgrade`` as well)::
-
-    factory SUFFIX .EXT
-    factory .EXT
-    factory
-
-When only an extension is given, the default suffix is retained. When no arguments
-are given, this signals that no factory (or sysupgrade) image exists.
-
-Aliases
-'''''''
-
-Sometimes multiple models use the same OpenWrt images. In this case, the ``alias``
-command can be used to create symlinks and additional entries in the autoupdater
-manifest for the alternative models.
-
-Standalone images
-'''''''''''''''''
-
-On targets without *per-device rootfs* support in OpenWrt, the commands described above
-can't be used. Instead, ``factory_image`` and ``sysupgrade_image`` are used::
-
-    factory_image PROFILE IMAGE .EXT
-    sysupgrade_image PROFILE IMAGE .EXT
-
-Again, the profile name must match the value printed by the aforementioned Lua
-command. The image name must match the part between the target name and the extension
-as generated by OpenWrt and is to be omitted when no such part exists.
-
-Packages
-''''''''
+The first argument is the device name in Gluon, which is part of the output
+image filename, and must correspond to the model string looked up by the
+autoupdater. The second argument is the corresponding device profile name in
+OpenWrt, as found in ``openwrt/target/linux/<target>/image/*``.
 
-The ``packages`` command takes an arbitrary number of arguments. Each argument
-defines an additional package to include in the images in addition to the default
-package sets defined by OpenWrt. When a package name is prefixed by a minus sign, the
-packages are excluded instead.
+A table of additional settings can be passed as a third argument:
 
-The ``packages`` command may be used at the top of a target definition to modify
-the default package list for all images, or just for a single device (when the
-target supports *per-default rootfs*).
+.. code-block:: lua
 
+  device('ubiquiti-edgerouter-x', 'ubnt_edgerouter-x', {
+    factory = false,
+    packages = {'-hostapd-mini'},
+    manifest_aliases = {
+      'ubnt-erx',
+    },
+  })
 
-Configuration
-'''''''''''''
+The supported additional settings are described in the following sections.
 
-The ``config`` command allows to add arbitrary target-specific OpenWrt configuration
-to be emitted to ``.config``.
-
-Notes
-'''''
-
-On devices with multiple WLAN adapters, care must also be taken that the primary MAC address is
-configured correctly. ``/lib/gluon/core/sysconfig/primary_mac`` should contain the MAC address which
-can be found on a label on most hardware; if it does not, ``/lib/gluon/upgrade/010-primary-mac``
-in ``gluon-core`` might need a fix. (There have also been cases in which the address was incorrect
-even on devices with only one WLAN adapter, in these cases a OpenWrt bug was the cause).
-
-
-Adding support for new hardware targets
----------------------------------------
-
-Adding a new target is much more complex than adding a new profile. There are two basic steps
-required for adding a new target:
-
-Package adjustments
-'''''''''''''''''''
-
-One package that may need adjustments for new targets is ``libplatforminfo`` (to be found in
-`packages/gluon/libs/libplatforminfo <https://github.com/freifunk-gluon/packages/tree/master/libs/libplatforminfo>`_).
-If the new platform works fine with the definitions found in ``default.c``, nothing needs to be done. Otherwise,
-create a definition for the added target or subtarget, either by symlinking one of the files in the ``templates``
-directory, or adding a new source file.
-
-On many targets, Gluon's network setup scripts (mainly in the package ``gluon-core``)
-won't run correctly without some adjustments, so better double check that everything is fine there (and the files
-``primary_mac``, ``lan_ifname`` and ``wan_ifname`` in ``/lib/gluon/core/sysconfig/`` contain sensible values).
-
-Build system support
-''''''''''''''''''''
-
-A definition for the new target must be created under ``targets``, and it must be added
-to ``targets/targets.mk``. The ``GluonTarget`` macro takes one to three arguments:
-the target name, the Gluon subtarget name (if the target has subtargets), and the
-OpenWrt subtarget name (if it differs from the Gluon subtarget). The third argument
-can be used to define multiple Gluon targets with different configuration for the
-same OpenWrt target, like it is done for the ``ar71xx-tiny`` target.
-
-After this, is should be sufficient to call ``make GLUON_TARGET=<target>`` to build the images for the new target.
+Suffixes and extensions
+~~~~~~~~~~~~~~~~~~~~~~~
+For many targets, OpenWrt generates images with the suffixes
+``-squashfs-factory.bin`` and ``-squashfs-sysupgrade.bin``. For devices with
+different image names, is it possible to override the suffixes and extensions
+using the settings ``factory``, ``factory_ext``, ``sysupgrade`` and
+``sysupgrade_ext``, for example:
+
+.. code-block:: lua
+
+  {
+    factory = '-squashfs-combined',
+    factory_ext = '.img.gz',
+    sysupgrade = '-squashfs-combined',
+    sysupgrade_ext = '.img.gz',
+  }
+
+Only settings that differ from the defaults need to be passed. ``factory`` and
+``sysupgrade`` can be set to ``false`` when no such images exist.
+
+For some device types, there are multiple factory images with different
+extensions. ``factory_ext`` can be set to a table of strings to account for this
+case:
+
+.. code-block:: lua
+
+  {
+    factory_ext = {'.img.gz', '.vmdk', '.vdi'},
+  }
+
+TODO: Extra images
+
+Aliases and manifest aliases
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Sometimes multiple devices exist that use the same OpenWrt images. To make it
+easier to find these images, the ``aliases`` setting can be used to define
+additional device names. Gluon will create symlinks for these names in the
+image output directory.
+
+.. code-block:: lua
+
+  device('aruba-ap-303', 'aruba_ap-303', {
+    factory = false,
+    aliases = {'aruba-instant-on-ap11'},
+  })
+
+The aliased name will also be added to the autoupdater manifest, allowing upgrade
+images to be found under the different name on targets that perform model name
+detection at runtime.
+
+It is also possible to add alternative names to the autoupdater manifest without
+creating a symlink by using ``manifest_aliases`` instead of ``aliases``, which
+should be done when the alternative name does not refer to a separate device.
+This is particularly useful to allow the autoupdater to work when the model name
+changed between Gluon versions.
+
+Package lists
+~~~~~~~~~~~~~
+Gluon generates lists of packages that are installed in all images based on a
+default list and the features and packages specified in the site configuration.
+
+In addition, OpenWrt defines additional per-device package lists. These lists
+may be modified in Gluon's device definitions, for example to include additional
+drivers and firmware, or to remove unneeded software. Packages to remove are
+prefixed with a ``-`` character.
+
+For many ath10k-based devices, this is used to replace the "CT" variant of
+ath10k with the mainline-based version:
+
+.. code-block:: lua
+
+  local ATH10K_PACKAGES_QCA9880 = {
+    'kmod-ath10k',
+    '-kmod-ath10k-ct',
+    '-kmod-ath10k-ct-smallbuffers',
+    'ath10k-firmware-qca988x',
+    '-ath10k-firmware-qca988x-ct',
+  }
+  device('openmesh-a40', 'openmesh_a40', {
+    packages = ATH10K_PACKAGES_QCA9880,
+    factory = false,
+  })
+
+This example also shows how to define a local variable, allowing the package
+list to be reused for multiple devices.
+
+Device flags
+~~~~~~~~~~~~
+
+The settings ``class``, ``deprecated`` or ``broken`` should be set according to
+the device support status. The default values are as follows:
+
+.. code-block:: lua
+
+  {
+    class = 'standard',
+    deprecated = false,
+    broken = false,
+  }
+
+- Device classes are described in :ref:`device-class-definition`
+- Broken devices are untested or do not meet our requirements as given by the
+  device checklist
+- Deprecated devices are slated for removal in a future Gluon version due to
+  hardware constraints
+
+Global settings
+~~~~~~~~~~~~~~~
+There is a number of directives that can be used outside of a ``device()``
+definition:
+
+- ``include('filename')``: Include another file with global settings
+- ``config(key, value)``: Set a config symbol in OpenWrt's ``.config``. Value
+  may be a string, number, boolean, or nil. Booleans and nil are used for
+  tristate symbols, where nil sets the symbol to ``m``.
+- ``try_config(key, value)``: Like ``config()``, but do not fail if setting
+  the symbol is not possible (usually because its dependencies are not met)
+- ``packages { 'package1', '-package2', ... }``: Define a list of packages to
+  add or remove for all devices of a target. Package lists passed to multiple
+  calls of ``packages`` will be aggregated.
+- ``defaults { key = value, ... }``: Set default values for any of the
+  additional settings that can be passed to ``device()``.
+
+Helper functions
+~~~~~~~~~~~~~~~~
+The following helpers can be used in the target configuration:
+
+- ``env.KEY`` allows to access environment variables
+- ``istrue(value)`` returns true if the passed string is a positive number
+  (often used with ``env``, for example ``if istrue(env.GLUON_DEBUG) then ...``)
+
+Hardware support in packages
+----------------------------
+In addition to the target configuration files, some device-specific changes may
+be required in packages.
+
+gluon-core
+~~~~~~~~~~
+- ``/lib/gluon/upgrade/010-primary-mac``: Override primary MAC address selection
+
+  Usually, the primary (label) MAC address is defined in OpenWrt's Device Trees.
+  For devices or targets where this is not the case, it is possible to specify
+  what interface to take the primary MAC address from in ``010-primary-mac``.
+
+- ``/lib/gluon/upgrade/020-interfaces``: Override LAN/WAN interface assignment
+
+  On PoE-powered devices, the PoE input port should be "WAN".
+
+- ``/usr/lib/lua/gluon/platform.lua``: Contains a list of outdoor devices
+
+gluon-setup-mode
+~~~~~~~~~~~~~~~~
+- ``/lib/gluon/upgrade/320-setup-ifname``: Contains a list of devices that use
+  the WAN port for the config mode
+
+  On PoE-powered devices, the PoE input port should be used for the config
+  mode. This is handled correctly by default for outdoor devices listed in
+  ``platform.lua``.
+
+libplatforminfo
+~~~~~~~~~~~~~~~
+When adding support for a new target to Gluon, it may be necessary to adjust
+libplatforminfo to define how autoupdater image names are derived from the
+model name.

docs/dev/mac_addresses.rst

@@ -10,9 +10,9 @@ Gluon tries to solve this issue by using a hash of the primary MAC address as a
 
 * 0: client0; WAN
 * 1: mesh0
-* 2: ibss0
+* 2: owe0
 * 3: wan_radio0 (private WLAN); batman-adv primary address
 * 4: client1; LAN
 * 5: mesh1
-* 6: ibss1
+* 6: owe1
 * 7: wan_radio1 (private WLAN); mesh VPN

docs/dev/packages.rst

@@ -3,6 +3,88 @@ Package development
 
 Gluon packages are OpenWrt packages and follow the same rules described at https://openwrt.org/docs/guide-developer/packages.
 
+Development workflow
+====================
+
+When you are developing packages, it often happens that you iteratively want to deploy
+and verify the state your development. There are two ways to verify your changes:
+
+1)
+  One way is to rebuild the complete firmware, flash it, configure it and verify your
+  development then. This usually takes at least a few minutes to get your changes
+  working so you can test them. Especially if you iterate a lot, this becomes tedious.
+
+2)
+  Another way is to rebuild only the package you are currently working on and
+  to deploy this package to your test system. Here not even a reboot is required.
+  This makes iterating relatively fast. Your test system could be real hardware or
+  even a qemu in most cases.
+
+Gluon provides scripts to enhance workflow 2). Here is an example illustrating
+the workflow using these scripts:
+
+.. code-block:: shell
+
+  # start a local qemu instance
+  contrib/run_qemu.sh output/images/factory/[...]-x86-64.img
+
+  # apply changes to the desired package
+  vi package/gluon-ebtables/files/etc/init.d/gluon-ebtables
+
+  # rebuild and push the package to the qemu instance
+  contrib/push_pkg.sh package/gluon-ebtables/
+
+  # test your changes
+  ...
+
+  # do more changes
+  ...
+
+  # rebuild and push the package to the qemu instance
+  contrib/push_pkg.sh package/gluon-ebtables/
+
+  # test your changes
+  ...
+
+  (and so on...)
+
+  # see help of the script for more information
+  contrib/push_pkg.sh -h
+  ...
+
+Features of ``push_pkg.sh``:
+
+* Works with compiled and non-compiled packages.
+
+  * This means it can be used in the development of C-code, Lua-Code and mostly any other code.
+
+* Works with native OpenWrt and Gluon packages.
+* Pushes to remote machines or local qemu instances.
+* Pushes multiple packages in one call if desired.
+* Performs site.conf checks.
+
+Implementation details of ``push_pkg.sh``:
+
+* First, the script builds an opkg package using the OpenWrt build system.
+* This package is pushed to a *target machine* using scp:
+
+  * By default the *target machine* is a locally running x86 qemu started using ``run_qemu.sh``.
+  * The *target machine* can also be remote machine. (See the cli switch ``-r``)
+  * Remote machines are not limited to a specific architecture. All architectures supported by gluon can be used as remote machines.
+
+* Finally opkg is used to install/update the packages in the target machine.
+
+  * While doing this, it will not override ``/etc/config`` with package defaults by default. (See the cli switch ``-P``).
+  * While doing this, opkg calls the ``check_site.lua`` from the package as post_install script to validate the ``site.conf``. This means that the ``site.conf`` of the target machine is used for this validation.
+
+Note that:
+
+* ``push_pkg.sh`` does neither build nor push dependencies of the packages automatically. If you want to update dependencies, you must explicitly specify them to be pushed.
+* If you add new packages, you must run ``make update config GLUON_TARGET=...``.
+* You can change the gluon target of the target machine via ``make config GLUON_TARGET=...``.
+* If you want to update the ``site.conf`` of the target machine, use ``push_pkg.sh package/gluon-site/``.
+* Sometimes when things break, you can heal them by compiling a package with its dependencies: ``cd openwrt; make package/gluon-ebtables/clean; make package/gluon-ebtables/compile; cd ..``.
+* You can exit qemu by pressing ``CTRL + a`` and ``c`` afterwards.
 
 Gluon package makefiles
 =======================
@@ -71,44 +153,63 @@ Feature flags
 =============
 
 Feature flags provide a convenient way to define package selections without
-making it necessary to list each package explicitly.
+making it necessary to list each package explicitly. The list of features to
+enable for a Gluon build is determined by the evaluated image-customization.lua file
+in the root-directory of the Site repository.
 
 The main feature flag definition file is ``package/features``, but each package
 feed can provide additional definitions in a file called ``features`` at the root
 of the feed repository.
 
-Each flag *$flag* without any explicit definition will simply include the package
-with the name *gluon-$flag* by default. The feature definition file can modify
-the package selection in two ways:
+Each flag *$flag* will include the package the name *gluon-$flag* by default.
+The feature definition file can modify the package selection by adding or removing
+packages when certain combinations of flags are set.
+
+Feature definitions use Lua syntax. Two basic functions are defined:
+
+* *feature(name, pkgs)*: Defines a new feature. *feature()* expects a feature
+  (flag) name and a list of packages to add or remove when the feature is
+  enabled.
+
+  * Defining a feature using *feature* replaces the default definition of
+    just including *gluon-$flag*.
+  * A package is removed when the package name is prefixed with a ``-`` (after
+    the opening quotation mark).
 
-* The *nodefault* function suppresses default of including the *gluon-$flag*
-  package
-* The *packages* function adds a list of packages (or removes, when package
-  names are prepended with minus signs) when a given logical expression
-  is satisfied
+* *when(expr, pkgs)*: Adds or removes packages when a given logical expression
+  of feature flags is satisfied.
+
+  * *expr* is a logical expression composed of feature flag names (each prefixed
+    with an underscore before the opening quotation mark), logical operators
+    (*and*, *or*, *not*) and parentheses.
+  * Referencing a feature flag in *expr* has no effect on the default handling
+    of the flag. When no *feature()* entry for a flag exists, it will still
+    add *gluon-$flag* by default.
+  * *pkgs* is handled as for *feature()*.
 
 Example::
 
-    nodefault 'web-wizard'
+    feature('web-wizard', {
+      'gluon-config-mode-hostname',
+      'gluon-config-mode-geo-location',
+      'gluon-config-mode-contact-info',
+      'gluon-config-mode-outdoor',
+    })
+
+    when(_'web-wizard' and _'mesh-vpn-fastd' or _'mesh-vpn-wireguard'), {
+      'gluon-config-mode-mesh-vpn',
+    })
 
-    packages 'web-wizard' \
-      'gluon-config-mode-hostname' \
-      'gluon-config-mode-geo-location' \
-      'gluon-config-mode-contact-info'
+    feature('no-radvd', {
+      '-gluon-radvd',
+    })
 
-    packages 'web-wizard & (mesh-vpn-fastd | mesh-vpn-tunneldigger)' \
-      'gluon-config-mode-mesh-vpn'
 
 This will
 
-* disable the inclusion of a (non-existent) package called *gluon-web-wizard*
-* enable three config mode packages when the *web-wizard* feature is enabled
+* disable the inclusion of the (non-existent) packages *gluon-web-wizard* and *gluon-no-radvd* when their
+  corresponding feature flags are evaluated as selected in the image-customization.lua file
+* enable four additional config mode packages when the *web-wizard* feature is enabled
 * enable *gluon-config-mode-mesh-vpn* when both *web-wizard* and one
-  of *mesh-vpn-fastd* and *mesh-vpn-tunneldigger* are enabled
-
-Supported syntax elements of logical expressions are:
-
-* \& (and)
-* \| (or)
-* \! (not)
-* parentheses
+  of *mesh-vpn-fastd* and *mesh-vpn-wireguard* are enabled
+* disable the *gluon-radvd* package when *gluon-no-radvd* is enabled

docs/dev/wan.rst → docs/dev/uplink.rst

-WAN support
-===========
+Uplink support
+==============
 
 As the WAN port of a node will be connected to a user's private network, it
 is essential that the node only uses the WAN when it is absolutely necessary.
@@ -11,6 +11,12 @@ There are two cases in which the WAN port is used:
 After the VPN connection has been established, the node should be able to reach
 the mesh's DNS servers and use these for all other name resolution.
 
+If a device has only a single Ethernet port (or group of ports), it will be
+used as an uplink port even when it is not labeled as "WAN" by default. This
+behavior can be controlled using the ``interfaces.single.default_roles``
+site.conf option. It is also possible to alter the interface assignment after
+installation by modifying ``/etc/config/gluon`` and running
+``gluon-reconfigure``.
 
 Routing tables
 ~~~~~~~~~~~~~~

docs/dev/web/config-mode.rst

@@ -11,18 +11,28 @@ gluon-config-mode-core
 gluon-config-mode-hostname
     Provides a hostname field.
 
-gluon-config-mode-autoupdater
+:doc:`gluon-config-mode-autoupdater <../../features/autoupdater>`
     Informs whether the autoupdater is enabled.
 
-gluon-config-mode-mesh-vpn
-    Allows toggling of mesh-vpn-fastd and setting a bandwidth limit.
+:doc:`gluon-config-mode-mesh-vpn <../../features/vpn>`
+    Allows toggling of installed mesh-vpn technology and setting a bandwidth limit.
 
 gluon-config-mode-geo-location
     Enables the user to set the geographical location of the node.
 
+:doc:`../../package/gluon-config-mode-geo-location-osm`
+    Lets the user click on a map to select the geographical location through a OSM map
+
 gluon-config-mode-contact-info
     Adds a field where the user can provide contact information.
 
+:doc:`../../package/gluon-web-cellular`
+    Adds advanced options to enter WWAN config.
+
+:doc:`../../package/gluon-web-network`
+    Adds option to configure used role on interfaces
+
+Most of the configuration options are described in :ref:`user-site-config_mode`
 
 Writing Config Mode modules
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/dev/web/controller.rst

@@ -74,8 +74,7 @@ Useful functions:
   - *header* (*key*, *value*): Adds an HTTP header to the reply to be sent to
     the client. Has no effect when non-header data has already been written.
   - *prepare_content* (*mime*): Sets the *Content-Type* header to the given MIME
-    type, potentially setting additional headers or modifying the MIME type to
-    accommodate browser quirks
+    type
   - *write* (*data*, ...): Sends the given data to the client. If headers have not
     been sent, it will be done before the data is written.
 

docs/dev/web/model.rst

@@ -26,7 +26,7 @@ Let's start with an example:
 
   return f
 
-The toplevel element of a model is always a *Form*, but it is also possible for
+The top-level element of a model is always a *Form*, but it is also possible for
 a model to return multiple forms, which are displayed one below the other.
 
 A *Form* has one or more *Sections*, and each *Section* has different types

docs/features/autoupdater.rst

@@ -7,8 +7,11 @@ Building Images
 ---------------
 
 By default, the autoupdater is disabled (as it is usually not helpful to have unexpected updates
-during development), but it can be enabled by setting the variable GLUON_BRANCH when building
-to override the default branch set in the site configuration.
+during development), but it can be enabled by setting the variable ``GLUON_AUTOUPDATER_ENABLED`` to ``1`` when building.
+It is also possible to override the default branch during build using the variable ``GLUON_AUTOUPDATER_BRANCH``.
+
+If a default branch is set neither in *site.conf* nor via ``GLUON_AUTOUPDATER_BRANCH``, the default branch is
+implementation-defined. Currently, the branch with the first name in alphabetical order is chosen.
 
 A manifest file for the updater can be generated with `make manifest`. A signing script (using
 ``ecdsautils``) can be found in the `contrib` directory. When creating the manifest, the
@@ -27,20 +30,42 @@ in ``site.mk``, care must be taken to pass the same ``GLUON_RELEASE`` to ``make
 as otherwise the generated manifest will be incomplete.
 
 
+Manifest format
+---------------
+
+The manifest starts with a short header, followed by the list of firmwares and signatures.
+The header contains the following information:
+
+.. code-block:: sh
+
+    BRANCH=stable
+    DATE=2020-10-07 00:00:00+02:00
+    PRIORITY=7
+
+- ``BRANCH`` is the autoupdater branch name that needs to match the nodes configuration.
+- ``DATE`` specifies when the time period for the update begins. Nodes will do their regular update during a random minute
+  between 4:00 and 4:59 am. Nodes might not always have a reliable NTP synchronization, which is why a fallback mechanism
+  exists, that checks for an update, and will execute if ``DATE`` is at least 24h in the past.
+- ``PRIORITY`` can be configured as ``GLUON_PRIORITY`` when generating the manifest or in ``site.mk``, and defines
+  the number of days over which the update should be stretched out after ``DATE``. Nodes will calculate a probability
+  based on the time left to determine when to update.
+
+
 Automated nightly builds
 ------------------------
 
 A fully automated nightly build could use the following commands:
 
-::
+.. code-block:: sh
 
     git pull
-    (git -C site pull)
+    # git -C site pull
     make update
-    make clean GLUON_TARGET=ar71xx-generic
+    make clean GLUON_TARGET=ath79-generic
     NUM_CORES_PLUS_ONE=$(expr $(nproc) + 1)
-    make -j$NUM_CORES_PLUS_ONE GLUON_TARGET=ar71xx-generic GLUON_BRANCH=experimental GLUON_RELEASE=$GLUON_RELEASE
-    make manifest GLUON_BRANCH=experimental GLUON_RELEASE=$GLUON_RELEASE
+    make -j$NUM_CORES_PLUS_ONE GLUON_TARGET=ath79-generic GLUON_RELEASE=$GLUON_RELEASE \
+        GLUON_AUTOUPDATER_BRANCH=experimental GLUON_AUTOUPDATER_ENABLED=1
+    make manifest GLUON_RELEASE=$GLUON_RELEASE GLUON_AUTOUPDATER_BRANCH=experimental
     contrib/sign.sh $SECRETKEY output/images/sysupgrade/experimental.manifest
 
     rm -rf /where/to/put/this/experimental

docs/features/configmode.rst

@@ -14,10 +14,13 @@ Activating Config Mode
 ----------------------
 
 Config Mode is automatically entered at the first boot. You can re-enter
-Config Mode by pressing and holding the RESET/WPS button for about three
+Config Mode by pressing and holding the RESET/WPS/DECT button for about three
 seconds. The device should reboot (all LEDs will turn off briefly) and
 Config Mode will be available.
 
+If you have access to the console of the node, there is the
+``gluon-enter-setup-mode`` command, which reboots a node into Config Mode.
+
 
 Port Configuration
 ------------------
@@ -35,3 +38,17 @@ Accessing Config Mode
 Config Mode can be accessed at http://192.168.1.1. The node will offer DHCP
 to clients. Should this fail, you may assign an IP from 192.168.1.0/24 to
 your computer manually.
+
+.. image:: configmode.png
+
+Advanced Config Options
+-----------------------
+
+Depending on the installed packages, the advanced config mode allows to configure packages further.
+
+* :doc:`gluon-web-wifi-config enable <wlan-configuration>` radios used for wifi and mesh as well as outdoor mode
+* :doc:`../package/gluon-web-network` allows to configure the used roles (uplink, mesh, client) on each interface
+* :doc:`../package/gluon-web-admin` allows to enter SSH keys or set a password in the `Remote access` section
+* :doc:`../package/gluon-web-cellular` allows to configure SIM card / WWAN settings on supported cellular devices
+
+The advanced config does also allow to upload a sysupgrade file to update the firmware to a different version.