Commit 9600749f authored by lemoer's avatar lemoer Committed by Martin Weinelt

contrib: Set up continuous integration through Jenkins

(cherry picked from commit 174dd314)
parent 52a1df09
pipeline {
agent { label 'gluon-docker' }
environment {
GLUON_SITEDIR = "contrib/ci/minimal-site"
GLUON_TARGET = "x86-64"
BUILD_LOG = "1"
}
stages {
stage('lint') {
steps {
sh 'luacheck package scripts targets'
}
}
stage('docs') {
steps {
sh 'make -C docs html'
}
}
stage('build') {
steps {
sh 'make update'
sh 'test -d /dl_cache && ln -s /dl_cache openwrt/dl || true'
sh 'make -j$(nproc) V=s'
}
}
}
}
FROM gluon
USER root
# this is needed to install default-jre-headless in debian slim images
RUN mkdir -p /usr/share/man/man1
RUN apt-get update && apt-get install -y default-jre-headless curl python3 python3-pip python3-sphinx git
RUN pip3 install jenkins-webapi sphinx_rtd_theme
# Get docker-compose in the agent container
RUN mkdir -p /home/jenkins
RUN mkdir -p /var/lib/jenkins
RUN mkdir -p /remoting
RUN chown gluon /home/jenkins
RUN chown gluon /var/lib/jenkins
RUN chown gluon /remoting
# Start-up script to attach the slave to the master
ADD slave.py /var/lib/jenkins/slave.py
USER gluon
WORKDIR /home/jenkins
ENV JENKINS_URL "https://build.ffh.zone/"
ENV JENKINS_SLAVE_ADDRESS ""
ENV SLAVE_EXECUTORS "1"
ENV SLAVE_LABELS "docker"
ENV SLAVE_WORING_DIR ""
ENV CLEAN_WORKING_DIR "true"
CMD [ "python3", "-u", "/var/lib/jenkins/slave.py" ]
# Gluon CI using Jenkins
## Requirements
- Only a host with docker.
## Architecture
![Screenshot from 2019-09-24 00-20-32](https://user-images.githubusercontent.com/601153/65468827-9edf2c80-de65-11e9-9fe0-56c3487719c3.png)
## Installation
You can support the gluon CI with your infrastructure:
1. You need to query @lemoer (freifunk@irrelefant.net) for credentials.
2. He will give you a `SLAVE_NAME` and a `SLAVE_SECRET` for your host.
3. Then go to your docker host and substitute the values for `SLAVE_NAME` and a `SLAVE_SECRET` in the following statements:
``` shell
git clone https://github.com/freifunk-gluon/gluon/
cd gluon/contrib/ci/jenkins-community-slave/
docker build -t gluon-jenkins .
mkdir /var/cache/openwrt_dl_cache/
docker run --detach --restart always \
-e "SLAVE_NAME=whoareyou" \
-e "SLAVE_SECRET=changeme" \
-v /var/cache/openwrt_dl_cache/:/dl_cache
```
4. Check whether the instance is running correctly:
- Your node should appear [here](https://build.ffh.zone/label/gluon-docker/).
- When clicking on it, Jenkins should state "Agent is connected." like here:
![Screenshot from 2019-09-24 01-00-52](https://user-images.githubusercontent.com/601153/65469209-dac6c180-de66-11e9-9d62-0d1c3b6b940b.png)
5. **Your docker container needs to be rebuilt, when the build dependencies of gluon change. So please be aware of that and update your docker container in that case.**
## Backoff
- If @lemoer is not reachable, please be patient at first if possible. Otherwise contact info@hannover.freifunk.net or join the channel `#freifunkh` on hackint.
from jenkins import Jenkins, JenkinsError, NodeLaunchMethod
import os
import signal
import sys
import urllib.request
import subprocess
import shutil
import requests
import time
slave_jar = '/var/lib/jenkins/slave.jar'
slave_name = os.environ['SLAVE_NAME'] if os.environ['SLAVE_NAME'] != '' else 'docker-slave-' + os.environ['HOSTNAME']
jnlp_url = os.environ['JENKINS_URL'] + '/computer/' + slave_name + '/slave-agent.jnlp'
slave_jar_url = os.environ['JENKINS_URL'] + '/jnlpJars/slave.jar'
print(slave_jar_url)
process = None
def clean_dir(dir):
for root, dirs, files in os.walk(dir):
for f in files:
os.unlink(os.path.join(root, f))
for d in dirs:
shutil.rmtree(os.path.join(root, d))
def slave_create(node_name, working_dir, executors, labels):
j = Jenkins(os.environ['JENKINS_URL'], os.environ['JENKINS_USER'], os.environ['JENKINS_PASS'])
j.node_create(node_name, working_dir, num_executors = int(executors), labels = labels, launcher = NodeLaunchMethod.JNLP)
def slave_delete(node_name):
j = Jenkins(os.environ['JENKINS_URL'], os.environ['JENKINS_USER'], os.environ['JENKINS_PASS'])
j.node_delete(node_name)
def slave_download(target):
if os.path.isfile(slave_jar):
os.remove(slave_jar)
loader = urllib.request.URLopener()
loader.retrieve(os.environ['JENKINS_URL'] + '/jnlpJars/slave.jar', '/var/lib/jenkins/slave.jar')
def slave_run(slave_jar, jnlp_url):
params = [ 'java', '-jar', slave_jar, '-jnlpUrl', jnlp_url ]
if os.environ['JENKINS_SLAVE_ADDRESS'] != '':
params.extend([ '-connectTo', os.environ['JENKINS_SLAVE_ADDRESS' ] ])
if os.environ['SLAVE_SECRET'] == '':
params.extend([ '-jnlpCredentials', os.environ['JENKINS_USER'] + ':' + os.environ['JENKINS_PASS'] ])
else:
params.extend([ '-secret', os.environ['SLAVE_SECRET'] ])
return subprocess.Popen(params, stdout=subprocess.PIPE)
def signal_handler(sig, frame):
if process != None:
process.send_signal(signal.SIGINT)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
def h():
print("ERROR!: please specify environment variables")
print("")
print('docker run -e "SLAVE_NAME=test" -e "SLAVE_SECRET=..." jenkins')
if os.environ.get('SLAVE_NAME') is None:
h()
sys.exit(1)
if os.environ.get('SLAVE_SECRET') is None:
h()
sys.exit(1)
def master_ready(url):
try:
r = requests.head(url, verify=False, timeout=None)
return r.status_code == requests.codes.ok
except:
return False
while not master_ready(slave_jar_url):
print("Master not ready yet, sleeping for 10sec!")
time.sleep(10)
slave_download(slave_jar)
print('Downloaded Jenkins slave jar.')
if os.environ['SLAVE_WORING_DIR']:
os.setcwd(os.environ['SLAVE_WORING_DIR'])
if os.environ['CLEAN_WORKING_DIR'] == 'true':
clean_dir(os.getcwd())
print("Cleaned up working directory.")
if os.environ['SLAVE_NAME'] == '':
slave_create(slave_name, os.getcwd(), os.environ['SLAVE_EXECUTORS'], os.environ['SLAVE_LABELS'])
print('Created temporary Jenkins slave.')
process = slave_run(slave_jar, jnlp_url)
print('Started Jenkins slave with name "' + slave_name + '" and labels [' + os.environ['SLAVE_LABELS'] + '].')
process.wait()
print('Jenkins slave stopped.')
if os.environ['SLAVE_NAME'] == '':
slave_delete(slave_name)
print('Removed temporary Jenkins slave.')
../../../docs/site-example/i18n/
\ No newline at end of file
../../../docs/site-example/modules
\ No newline at end of file
-- This is an example site configuration for Gluon v2018.2+
--
-- Take a look at the documentation located at
-- https://gluon.readthedocs.io/ for details.
--
-- This configuration will not work as is. You're required to make
-- community specific changes to it!
{
-- Used for generated hostnames, e.g. freifunk-abcdef123456. (optional)
-- hostname_prefix = 'freifunk-',
-- Name of the community.
site_name = 'Continious Integration',
-- Shorthand of the community.
site_code = 'ci',
-- 32 bytes of random data, encoded in hexadecimal
-- This data must be unique among all sites and domains!
-- Can be generated using: echo $(hexdump -v -n 32 -e '1/1 "%02x"' </dev/urandom)
domain_seed = 'e9608c4ff338b920992d629190e9ff11049de1dfc3f299eac07792dfbcda341c',
-- Prefixes used within the mesh.
-- prefix6 is required, prefix4 can be omitted if next_node.ip4
-- is not set.
prefix4 = '10.0.0.0/20',
prefix6 = 'fd::/64',
-- Timezone of your community.
-- See https://openwrt.org/docs/guide-user/base-system/system_configuration#time_zones
timezone = 'CET-1CEST,M3.5.0,M10.5.0/3',
-- List of NTP servers in your community.
-- Must be reachable using IPv6!
-- ntp_servers = {'1.ntp.services.ffxx'},
-- Wireless regulatory domain of your community.
regdom = 'DE',
-- Wireless configuration for 2.4 GHz interfaces.
wifi24 = {
-- Wireless channel.
channel = 1,
-- ESSID used for client network.
ap = {
ssid = 'gluon-ci-ssid',
-- disabled = true, -- (optional)
},
mesh = {
-- Adjust these values!
id = 'ueH3uXjdp', -- usually you don't want users to connect to this mesh-SSID, so use a cryptic id that no one will accidentally mistake for the client WiFi
mcast_rate = 12000,
-- disabled = true, -- (optional)
},
},
-- Wireless configuration for 5 GHz interfaces.
-- This should be equal to the 2.4 GHz variant, except
-- for channel.
wifi5 = {
channel = 44,
outdoor_chanlist = '100-140',
ap = {
ssid = 'gluon-ci-ssid',
},
mesh = {
-- Adjust these values!
id = 'ueH3uXjdp',
mcast_rate = 12000,
},
},
-- The next node feature allows clients to always reach the node it is
-- connected to using a known IP address.
next_node = {
-- anycast IPs of all nodes
-- name = { 'nextnode.location.community.example.org', 'nextnode', 'nn' },
ip4 = '10.0.0.1',
ip6 = 'fd::1',
},
mesh = {
vxlan = true,
batman_adv = {
routing_algo = 'BATMAN_IV'
}
},
mesh_vpn = {
-- enabled = true,
mtu = 1312,
fastd = {
-- Refer to https://fastd.readthedocs.io/en/latest/ to better understand
-- what these options do.
-- List of crypto-methods to use.
methods = {'salsa2012+umac'},
-- configurable = true,
-- syslog_level = 'warn',
groups = {
backbone = {
-- Limit number of connected peers to reduce bandwidth.
limit = 1,
-- List of peers.
peers = {
},
},
},
},
bandwidth_limit = {
-- The bandwidth limit can be enabled by default here.
enabled = false,
-- Default upload limit (kbit/s).
egress = 200,
-- Default download limit (kbit/s).
ingress = 3000,
},
},
autoupdater = {
-- Default branch. Don't forget to set GLUON_BRANCH when building!
branch = 'stable',
-- List of branches. You may define multiple branches.
branches = {
stable = {
name = 'stable',
-- List of mirrors to fetch images from. IPv6 required!
mirrors = {'http://1.updates.services.ffhl/stable/sysupgrade'},
-- Number of good signatures required.
-- Have multiple maintainers sign your build and only
-- accept it when a sufficient number of them have
-- signed it.
good_signatures = 2,
-- List of public keys of maintainers.
pubkeys = {
},
},
},
},
}
../../../docs/site-example/site.mk
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment