Merge pull request #3054 from blocktrron/flexible-feature-packages

build: implement flexible site-selection system

Makefile

@@ -35,6 +35,14 @@ ifneq ($(GLUON_BRANCH),)
   GLUON_AUTOUPDATER_ENABLED ?= 1
 endif
 
+ifneq ($(GLUON_FEATURES)$(GLUON_FEATURES_standard)$(GLUON_FEATURES_tiny),)
+  $(error *** Warning: GLUON_FEATURES has been obsolete, please use the image-customization.lua file instead.)
+endif
+
+ifneq ($(GLUON_SITE_PACKAGES)$(GLUON_SITE_PACKAGES_standard)$(GLUON_SITE_PACKAGES_tiny),)
+  $(error *** Warning: GLUON_SITE_PACKAGES has been obsolete, please use the image-customization.lua file instead.)
+endif
+
 GLUON_AUTOUPDATER_ENABLED ?= 0
 
 # initialize (possibly already user set) directory variables

contrib/ci/minimal-site/image-customization.lua

+features {
+	'autoupdater',
+	'ebtables-filter-multicast',
+	'ebtables-filter-ra-dhcp',
+	'ebtables-limit-arp',
+	'mesh-batman-adv-15',
+	'mesh-vpn-fastd',
+	'respondd',
+	'status-page',
+	'web-advanced',
+	'web-wizard',
+}
+
+if not device_class('tiny') then
+	features {'wireless-encryption-wpa3'}
+end

contrib/ci/olsr-site/image-customization.lua

+features {
+	'autoupdater',
+	'ebtables-filter-multicast',
+	'ebtables-filter-ra-dhcp',
+	'ebtables-limit-arp',
+	'mesh-olsrd',
+	'mesh-vpn-fastd',
+	'respondd',
+	'status-page',
+	'web-advanced',
+	'web-wizard',
+}
+
+packages {
+	'iwinfo',
+}
+
+if not device_class('tiny') then
+	features {'wireless-encryption-wpa3'}
+end

contrib/ci/olsr-site/site.mk

 ##	gluon site.mk makefile example
 
-##	GLUON_FEATURES
-#		Specify Gluon features/packages to enable;
-#		Gluon will automatically enable a set of packages
-#		depending on the combination of features listed
-
-GLUON_FEATURES := \
-	autoupdater \
-	ebtables-filter-multicast \
-	ebtables-filter-ra-dhcp \
-	ebtables-limit-arp \
-	mesh-olsrd \
-	mesh-vpn-fastd \
-	respondd \
-	status-page \
-	web-advanced \
-	web-wizard
-
-GLUON_FEATURES_standard := \
-  wireless-encryption-wpa3
-
-##	GLUON_SITE_PACKAGES
-#		Specify additional Gluon/OpenWrt packages to include here;
-#		A minus sign may be prepended to remove a packages from the
-#		selection that would be enabled by default or due to the
-#		chosen feature flags
-
-GLUON_SITE_PACKAGES := iwinfo
-
 ##	DEFAULT_GLUON_RELEASE
 #		version string to use for images
 #		gluon relies on

docs/dev/packages.rst

@@ -154,7 +154,8 @@ Feature flags
 
 Feature flags provide a convenient way to define package selections without
 making it necessary to list each package explicitly. The list of features to
-enable for a Gluon build is set by the *GLUON_FEATURES* variable in *site.mk*.
+enable for a Gluon build is determined by the evaluated image-customization.lua file
+in the root-directory of the Site repository.
 
 The main feature flag definition file is ``package/features``, but each package
 feed can provide additional definitions in a file called ``features`` at the root
@@ -207,7 +208,7 @@ Example::
 This will
 
 * disable the inclusion of the (non-existent) packages *gluon-web-wizard* and *gluon-no-radvd* when their
-  corresponding feature flags appear in *GLUON_FEATURES*
+  corresponding feature flags are evaluated as selected in the image-customization.lua file
 * enable four additional config mode packages when the *web-wizard* feature is enabled
 * enable *gluon-config-mode-mesh-vpn* when both *web-wizard* and one
   of *mesh-vpn-fastd* and *mesh-vpn-tunneldigger* are enabled

docs/features/multidomain.rst

@@ -106,7 +106,7 @@ Via config mode
 ^^^^^^^^^^^^^^^
 
 To allow switching the domain via config mode, add ``config-mode-domain-select``
-to GLUON_FEATURES in site.mk.
+to the enabled features in the image-customization.lua file.
 
 |image0|
 

docs/features/private-wlan.rst

@@ -6,7 +6,7 @@ Please note that you should not enable Wired Mesh on the uplink port at the same
 
 The private WLAN is encrypted using WPA2 by default. On devices with enough flash and a supported radio,
 WPA3 or WPA2/WPA3 mixed-mode can be used instead of WPA2. For this to work, the ``wireless-encryption-wpa3``
-feature has to be added to ``GLUON_FEATURES``.
+feature has to be enabled as a feature.
 
 It is recommended to enable IEEE 802.11w management frame protection for WPA2/WPA3 networks, however this
 can lead to connectivity problems for older clients. In this case, management frame protection can be

docs/features/vpn.rst

@@ -8,7 +8,7 @@ Protocol handlers
 ^^^^^^^^^^^^^^^^^
 
 There are currently three protocol handlers which can be selected
-via ``GLUON_FEATURES`` in ``site.mk``:
+as a feature:
 
 mesh-vpn-fastd
 """"""""""""""

docs/multidomain-site-example/image-customization.lua

+features {
+	'autoupdater',
+	'ebtables-filter-multicast',
+	'ebtables-filter-ra-dhcp',
+	'ebtables-limit-arp',
+	'mesh-batman-adv-15',
+	'mesh-vpn-fastd',
+	'respondd',
+	'status-page',
+	'web-advanced',
+	'web-wizard',
+}
+
+packages {
+	'iwinfo',
+}
+
+if not device_class('tiny') then
+	features {'wireless-encryption-wpa3'}
+end

docs/multidomain-site-example/site.mk

 ##	gluon site.mk makefile example
 
-##	GLUON_FEATURES
-#		Specify Gluon features/packages to enable;
-#		Gluon will automatically enable a set of packages
-#		depending on the combination of features listed
-
-GLUON_FEATURES := \
-	autoupdater \
-	ebtables-filter-multicast \
-	ebtables-filter-ra-dhcp \
-	ebtables-limit-arp \
-	mesh-batman-adv-15 \
-	mesh-vpn-fastd \
-	respondd \
-	status-page \
-	web-advanced \
-	web-wizard
-
-GLUON_FEATURES_standard := \
-	wireless-encryption-wpa3
-
 ##	GLUON_MULTIDOMAIN
 #		Build gluon with multidomain support.
 
 GLUON_MULTIDOMAIN=1
 
-##	GLUON_SITE_PACKAGES
-#		Specify additional Gluon/OpenWrt packages to include here;
-#		A minus sign may be prepended to remove a packages from the
-#		selection that would be enabled by default or due to the
-#		chosen feature flags
-
-GLUON_SITE_PACKAGES := iwinfo
-
 ##	DEFAULT_GLUON_RELEASE
 #		version string to use for images
 #		gluon relies on

docs/package/gluon-ebtables-limit-arp.rst

@@ -25,5 +25,6 @@ This package is installed by default if the selected routing
 feature is *mesh-batman-adv-15*.
 It can be unselected via::
 
-    GLUON_SITE_PACKAGES := \
-      -gluon-ebtables-limit-arp
+    packages {
+      '-gluon-ebtables-limit-arp',
+    }

docs/site-example/image-customization.lua

+packages {'iwinfo'}
+
+features {
+	'autoupdater',
+	'ebtables-filter-multicast',
+	'ebtables-filter-ra-dhcp',
+	'ebtables-limit-arp',
+	'mesh-batman-adv-15',
+	'mesh-vpn-fastd',
+	'respondd',
+	'status-page',
+	'web-advanced',
+	'web-wizard'
+}
+
+if not device_class('tiny') then
+	features {
+		'wireless-encryption-wpa3'
+	}
+end

docs/site-example/site.mk

 ##	gluon site.mk makefile example
 
-##	GLUON_FEATURES
-#		Specify Gluon features/packages to enable;
-#		Gluon will automatically enable a set of packages
-#		depending on the combination of features listed
-
-GLUON_FEATURES := \
-	autoupdater \
-	ebtables-filter-multicast \
-	ebtables-filter-ra-dhcp \
-	ebtables-limit-arp \
-	mesh-batman-adv-15 \
-	mesh-vpn-fastd \
-	respondd \
-	status-page \
-	web-advanced \
-	web-wizard
-
-GLUON_FEATURES_standard := \
-  wireless-encryption-wpa3
-
-##	GLUON_SITE_PACKAGES
-#		Specify additional Gluon/OpenWrt packages to include here;
-#		A minus sign may be prepended to remove a packages from the
-#		selection that would be enabled by default or due to the
-#		chosen feature flags
-
-GLUON_SITE_PACKAGES := iwinfo
-
 ##	DEFAULT_GLUON_RELEASE
 #		version string to use for images
 #		gluon relies on

docs/user/site.rst

@@ -134,8 +134,8 @@ wifi24 \: optional
   For an OWE secured network, the ``owe_ssid`` string has to be set. It sets the
   SSID for the opportunistically encrypted wireless network, to which compatible
   clients can connect to.
-  For OWE to work, the ``wireless-encryption-wpa3`` has to be enabled (usually by
-  adding it to ``GLUON_FEATURES_standard``) in your ``site.mk``.
+  For OWE to work, the ``wireless-encryption-wpa3`` has to be enabled as a feature
+  in your site.
   To utilize the OWE transition mode, ``owe_transition_mode`` has to be set to true.
   When ``owe_transition_mode`` is enabled, the OWE secured SSID will be hidden.
   Compatible devices will automatically connect to the OWE secured SSID when selecting
@@ -611,33 +611,6 @@ GLUON_DEPRECATED
   and ``upgrade`` for existing configurations (where upgrades for existing
   deployments of low-flash devices are required). Defaults to ``0``.
 
-GLUON_FEATURES
-  Defines a list of features to include. Depending on the device, the feature list
-  defined from this value is combined with the feature list for either the standard
-  or the tiny device-class. The resulting feature list is used to generate the default
-  package set.
-
-GLUON_FEATURES_standard
-  Defines a list of additional features to include or exclude for devices of
-  the standard device-class.
-
-GLUON_FEATURES_tiny
-  Defines a list of additional features to include or exclude for devices of
-  the tiny device-class.
-
-GLUON_SITE_PACKAGES
-  Defines a list of packages which should be installed in addition to the
-  default package set. It is also possible to remove packages from the
-  default set by prepending a minus sign to the package name.
-
-GLUON_SITE_PACKAGES_standard
-  Defines a list of additional packages to include or exclude for devices of
-  the standard device-class.
-
-GLUON_SITE_PACKAGES_tiny
-  Defines a list of additional packages to include or exclude for devices of
-  the tiny device-class.
-
 GLUON_RELEASE
   The current release version Gluon should use.
 
@@ -674,8 +647,8 @@ leading to entangled package names like *gluon-mesh-vpn-fastd-respondd* or
 *gluon-status-page-mesh-batman-adv-i18n-de*.
 
 For this reason, we have introduced *feature flags*, which can be specified
-in the *GLUON_FEATURES* variable. These flags allow to specify a set of features
-on a higher level than individual package names.
+using the image-customization.lua file. These flags allow to specify
+a set of features on a higher level than individual package names.
 
 Most Gluon packages can simply be specified as feature flags by removing the ``gluon-``
 prefix: The feature flag corresponding to the package *gluon-mesh-batman-adv-15* is
@@ -697,9 +670,10 @@ flags using a flexible ruleset defined in the Gluon repo or site package feeds.
 To some extent, it will even allow us to further modularize existing Gluon packages,
 without necessitating changes to existing site configurations.
 
-It is still possible to override such automatic rules using *GLUON_SITE_PACKAGES*
-(e.g., ``-gluon-status-page-mesh-batman-adv`` to remove the automatically added
-package *gluon-status-page-mesh-batman-adv*).
+It is still possible to override such automatic rules by removing them using 
+*packages* in the image-customization.lua file
+(e.g., ``features { '-gluon-status-page-mesh-batman-adv' }`` to remove
+the automatically added package *gluon-status-page-mesh-batman-adv*).
 
 For convenience, there are two feature flags that do not directly correspond to a Gluon
 package:
@@ -707,19 +681,62 @@ package:
 * web-wizard
 
   Includes the *gluon-config-mode-...* base packages (hostname, geolocation and contact info),
-  as well as the *gluon-config-mode-autoupdater* (when *autoupdater* is in *GLUON_FEATURES*),
-  and *gluon-config-mode-mesh-vpn* (when *mesh-vpn-fastd* or *mesh-vpn-tunneldigger* are in
-  *GLUON_FEATURES*)
+  as well as the *gluon-config-mode-autoupdater* (when *autoupdater* is an enabled feature),
+  and *gluon-config-mode-mesh-vpn* (when *mesh-vpn-fastd* or *mesh-vpn-tunneldigger* are
+  enabled features)
 
 * web-advanced
 
   Includes the *gluon-web-...* base packages (admin, network, WiFi config),
-  as well as the *gluon-web-autoupdater* (when *autoupdater* is in *GLUON_FEATURES*)
+  as well as the *gluon-web-autoupdater* (when *autoupdater* is an enabled feature),
 
-We recommend to use *GLUON_SITE_PACKAGES* for non-Gluon OpenWrt packages only and
-completely rely on *GLUON_FEATURES* for Gluon packages, as it is shown in the
+We recommend to include packages for non-Gluon OpenWrt packages only and
+completely rely on features for Gluon packages, as it is shown in the
 example *site.mk*.
 
+.. _site-image-customization:
+
+Image customization
+^^^^^^^^^^^^^^^^^^^
+
+Gluon allows configuration of the build-parameters for the images. A Lua domain-specific-language
+has been implemented to facilitate this configuration. This file is interpreted for every device
+independently. The file is located in the site-repository root as ``image-config``.
+
+A simple example for a device-specific image configuration can be found in the site-example.
+
+The following functions are available:
+
+device(device_name_list)
+  Returns true in case the current device is in the list of devices specified in ``device_name_list``.
+  ``device_name_list`` is a table of strings.
+
+target(openwrt_target, openwrt_subtarget)
+  Returns true in case the current device is of the specified OpenWrt target and subtarget.
+  The parameter ```openwrt_subtarget``` is optional. If it is not specified, only the target is matched.
+
+device_class(dev_class)
+  Returns true in case the current device is of the specified device class.
+
+features(feature_table)
+  Includes the specified list of features in the image. ``feature_table`` is a table of strings.
+  These strings can be prefixed with a dash to exclude features included earlier in the file.
+
+packages(package_table)
+  Includes the specified list of packages in the image. ``package_table`` is a table of strings.
+  These strings can be prefixed with a dash to exclude packages included earlier in the file.
+
+broken(broken_state)
+  Overrides the broken state specified by Gluon. Can be used to mark a device as broken or
+  remove the pre-defined broken state.
+
+disable()
+  Disables image generation.
+
+disable_factory()
+  Disables factory image generation. Sysupgrade images are still generated and stored in the image
+  output directory.
+
 .. _site-config-mode-texts:
 
 Config mode texts

package/features

--- GLUON_FEATURES definition file
+-- Feature definition file
 --
 -- See the page `dev/packages` (Developer Documentation / Package development)
 -- in the `docs` directory or on gluon.readthedocs.io for information on the

scripts/image_customization_lib.lua

+local M = {
+	customization_file = nil,
+}
+
+local function evaluate_device(env, dev)
+	local selections = {
+		features = {},
+		packages = {},
+	}
+	local funcs = {}
+	local device_overrides = {}
+
+	local function add_elements(element_type, element_list)
+		-- We depend on the fact both feature and package
+		-- are already initialized as empty tables
+		for _, element in ipairs(element_list) do
+			table.insert(selections[element_type], element)
+		end
+	end
+
+	local function add_override(ovr_key, ovr_value)
+		device_overrides[ovr_key] = ovr_value
+	end
+
+	function funcs.features(features)
+		add_elements('features', features)
+	end
+
+	function funcs.packages(packages)
+		add_elements('packages', packages)
+	end
+
+	function funcs.broken(broken)
+		assert(
+			type(broken) == 'boolean',
+			'Incorrect use of broken(): has to be a boolean value')
+		add_override('broken', broken)
+	end
+
+	function funcs.disable()
+		add_override('disabled', true)
+	end
+
+	function funcs.disable_factory()
+		add_override('disable_factory', true)
+	end
+
+	function funcs.device(device_names)
+		assert(
+			type(device_names) == 'table',
+			'Incorrect use of device(): pass a list of device names as argument')
+
+		for _, device_name in ipairs(device_names) do
+			if device_name == dev.image then
+				return true
+			end
+		end
+
+		return false
+	end
+
+	function funcs.target(target, subtarget)
+		assert(
+			type(target) == 'string',
+			'Incorrect use of target(): pass a target name as first argument')
+
+		if target ~= env.BOARD then
+			return false
+		end
+
+		if subtarget and subtarget ~= env.SUBTARGET then
+			return false
+		end
+
+		return true
+	end
+
+	function funcs.device_class(class)
+		return dev.options.class == class
+	end
+
+	-- Evaluate the feature definition files
+	setfenv(M.customization_file, funcs)
+	M.customization_file()
+
+	return {
+		selections = selections,
+		device_overrides = device_overrides,
+	}
+end
+
+function M.get_selections(dev)
+	local return_object = {
+		features = {},
+		packages = {},
+	}
+
+	if M.customization_file == nil then
+		-- No customization file found
+		return return_object
+	end
+
+	local eval_result = evaluate_device(M.env, dev)
+	return eval_result.selections
+end
+
+function M.device_overrides(dev)
+	if M.customization_file == nil then
+		-- No customization file found
+		return {}
+	end
+
+	local eval_result = evaluate_device(M.env, dev)
+	return eval_result.device_overrides
+end
+
+function M.init(env)
+	local filename = env.GLUON_SITEDIR .. '/image-customization.lua'
+
+	M.env = env
+
+	local f, _ = loadfile(filename)
+	if not f then
+		-- No customization file found, nothing to do
+		return
+	end
+
+	M.customization_file = f
+end
+
+return M

scripts/target_config_lib.lua

 local lib = dofile('scripts/target_lib.lua')
 local feature_lib = dofile('scripts/feature_lib.lua')
+local image_customization_lib = dofile('scripts/image_customization_lib.lua')
 local env = lib.env
 
 local target = env.GLUON_TARGET
@@ -15,6 +16,8 @@ else
 	openwrt_config_target = env.BOARD
 end
 
+-- Initialize image-customization
+image_customization_lib.init(env)
 
 -- Split a string into words
 local function split(s)
@@ -73,23 +76,6 @@ local function file_exists(file)
 	return true
 end
 
-local function site_vars(var)
-	return lib.exec_capture_raw(string.format(
-[[
-MAKEFLAGS= make print _GLUON_SITE_VARS_=%s --no-print-directory -s -f - <<'END_MAKE'
-include $(GLUON_SITEDIR)/site.mk
-
-print:
-	echo -n '$(_GLUON_SITE_VARS_)'
-END_MAKE
-]],
-	lib.escape(var)))
-end
-
-local function site_packages(image)
-	return split(site_vars(string.format('$(GLUON_%s_SITE_PACKAGES)', image)))
-end
-
 local function feature_packages(features)
 	local files = {'package/features'}
 	for _, feed in ipairs(feeds) do
@@ -102,20 +88,29 @@ local function feature_packages(features)
 	return feature_lib.get_packages(files, features)
 end
 
--- This involves running a few processes to evaluate site.mk, so we add a simple cache
-local class_cache = {}
-local function class_packages(class)
-	if class_cache[class] then
-		return class_cache[class]
-	end
+local function site_specific_packages(dev_info)
+	local site_selections
+	local site_packages
+	local feature_inherited_pkgs
+	local site_features
+
+	-- Get all enabled selections from image-customization.lua
+	site_selections = image_customization_lib.get_selections(dev_info)
+
+	-- First read enabled features from site
+	site_features = site_selections['features']
+	site_features = compact_list(site_features, false)
+
+	-- Create List from packages inherited from features
+	feature_inherited_pkgs = feature_packages(site_features)
 
-	local features = site_vars(string.format('$(GLUON_FEATURES) $(GLUON_FEATURES_%s)', class))
-	features = compact_list(split(features), false)
+	-- Read list of packages from site
+	site_packages = site_selections['packages']
 
-	local pkgs = feature_packages(features)
-	pkgs = concat_list(pkgs, split(site_vars(string.format('$(GLUON_SITE_PACKAGES) $(GLUON_SITE_PACKAGES_%s)', class))))
+	-- Concat feature-packages with site-packages
+	local pkgs = concat_list(feature_inherited_pkgs, site_packages)
 
-	class_cache[class] = pkgs
+	-- Negations for the resulting package-list are dealt with in the calling function
 	return pkgs
 end
 
@@ -192,9 +187,8 @@ for _, dev in ipairs(lib.devices) do
 	end
 
 	handle_pkgs(lib.target_packages)
-	handle_pkgs(class_packages(dev.options.class))
 	handle_pkgs(dev.options.packages or {})
-	handle_pkgs(site_packages(dev.image))
+	handle_pkgs(site_specific_packages(dev))
 
 	local profile_config = string.format('%s_DEVICE_%s', openwrt_config_target, dev.name)
 	lib.config(

scripts/target_lib.lua

+local image_customization_lib = dofile('scripts/image_customization_lib.lua')
+
 -- Functions for use in targets/*
 local F = {}
 
@@ -29,6 +31,8 @@ M.configs = {}
 M.devices = {}
 M.images = {}
 
+-- Initialize image-customization
+image_customization_lib.init(env)
 
 local default_options = {
 	factory = '-squashfs-factory',
@@ -55,11 +59,33 @@ function F.istrue(v)
 	return (tonumber(v) or 0) > 0
 end
 
-local function want_device(dev, options)
-	if options.broken and not F.istrue(env.BROKEN) then
+local function device_broken(device_info, overrides)
+	if F.istrue(env.BROKEN) then
+		return false
+	end
+
+	if overrides['broken'] ~= nil then
+		return overrides['broken'] == true
+	elseif device_info.options.broken then
+		return true
+	end
+
+	return false
+end
+
+local function want_device(device_info)
+	local overrides = image_customization_lib.device_overrides(device_info)
+
+	-- Check if device is disabled via image-customization.lua in site
+	if overrides['disabled'] then
+		return false
+	end
+
+	if device_broken(device_info, overrides) then
 		return false
 	end
-	if options.deprecated and env.GLUON_DEPRECATED == '0' then
+
+	if device_info.options.deprecated and env.GLUON_DEPRECATED == '0' then
 		return false
 	end
 
@@ -67,13 +93,10 @@ local function want_device(dev, options)
 		return true
 	end
 
-	unknown_devices[dev] = nil
-	return gluon_devices[dev]
+	unknown_devices[device_info.image] = nil
+	return gluon_devices[device_info.image]
 end
 
-local full_deprecated = env.GLUON_DEPRECATED == 'full'
-
-
 local function merge(a, b)
 	local ret = {}
 	for k, v in pairs(a) do
@@ -210,18 +233,33 @@ local function as_table(v)
 	end
 end
 
-function F.device(image, name, options)
-	options = merge(default_options, options)
+local function disable_factory_image(device_info)
+	if device_info.options.deprecated and env.GLUON_DEPRECATED ~= 'full' then
+		return true
+	end
 
-	if not want_device(image, options) then
-		return
+	local overrides = image_customization_lib.device_overrides(device_info)
+	if overrides["disable_factory"] then
+		return true
 	end
 
-	table.insert(M.devices, {
+	return false
+end
+
+function F.device(image, name, options)
+	options = merge(default_options, options)
+
+	local device_info = {
 		image = image,
 		name = name,
 		options = options,
-	})
+	}
+
+	if not want_device(device_info) then
+		return
+	end
+
+	table.insert(M.devices, device_info)
 
 	if options.sysupgrade then
 		add_image {
@@ -236,7 +274,7 @@ function F.device(image, name, options)
 		}
 	end
 
-	if options.deprecated and not full_deprecated then
+	if disable_factory_image(device_info) then
 		return
 	end