Skip to content
Snippets Groups Projects
Commit 3bb3f02b authored by Martin Weinelt's avatar Martin Weinelt
Browse files

contrib: drop Jenkins setup

Has gone unused for a while now and if it is needed again it can be
retrieved from the history.
parent f193b75c
No related branches found
No related tags found
No related merge requests found
pipeline {
agent none
environment {
GLUON_SITEDIR = "contrib/ci/minimal-site"
GLUON_TARGET = "x86-64"
BUILD_LOG = "1"
}
stages {
stage('lint') {
parallel {
stage('lint-lua') {
agent { label 'gluon-docker' }
steps {
sh label: 'Identify runner', script: 'echo $SLAVE_NAME'
sh 'make lint-lua'
}
}
stage('lint-sh') {
agent { label 'gluon-docker-v1' }
steps {
sh label: 'Identify runner', script: 'echo $SLAVE_NAME'
sh 'make lint-sh'
}
}
}
}
stage('docs') {
agent { label 'gluon-docker' }
steps {
sh label: 'Identify runner', script: 'echo $SLAVE_NAME'
sh 'make -C docs html'
}
}
stage('build') {
agent { label 'gluon-docker-v2' }
steps {
sh label: 'Identify runner', script: 'echo $SLAVE_NAME'
sh 'make update'
sh 'test -d /dl_cache && ln -s /dl_cache openwrt/dl || true'
timeout(time: 2, unit: "HOURS") {
sh 'make -j$(nproc) V=s'
}
stash includes: '**/output/images/factory/*-x86-64.img.gz', name: 'gluon-x86-64-factory'
}
}
stage('test') {
agent { label 'gluon-vmx' }
steps {
sh label: 'Identify runner', script: 'echo $SLAVE_NAME'
unstash 'gluon-x86-64-factory'
sh label: 'Unpack image', script: 'gunzip -cd ./output/images/factory/*x86-64*.img.gz > ./image.img'
sh label: 'Print python environment', script: 'python3 -m pip freeze'
script {
for (f in findFiles(glob: 'tests/*.py')) {
timeout(time: 10, unit: "MINUTES") {
sh label: "Test ${f.name}", script: "python3 tests/${f.name} --use-tmp-workdir"
}
}
}
}
}
}
}
/*
api-history:
Every time the build dependencies of gluon change, the version
every container has to be rebuilt. Therefore, we use Jenkins
labels which intoduce a version number which is documented here.
As soon, as you properly rebuilt your docker container, you
can notify lemoer, that you have updated your node.
- gluon-docker-v1:
- add shellcheck binary to the build environment
- gluon-docker-v2:
- add qemu-testlab testing, requires KVM virtualization support
- require rsync dependency to be able to build the next branch
- gluon-vmx
- splits the qemu testing from the gluon-docker-v2 label to accomodate
nodes without the vmx cpu flag
*/
FROM gluonmesh/build:latest
USER root
# this is needed to install default-jre-headless in debian slim images
RUN mkdir -p /usr/share/man/man1
RUN apt-get update && apt-get install -y default-jre-headless curl git netcat-openbsd python3 python3-pip qemu-system-x86 iproute2 openssh-client rsync
RUN python3 -m pip install jenkins-webapi sphinx sphinx_rtd_theme gluon-qemu-testlab==0.0.5
# Get docker-compose in the agent container
RUN mkdir -p /home/jenkins
RUN mkdir -p /var/lib/jenkins
RUN mkdir -p /remoting
RUN chown gluon /home/jenkins
RUN chown gluon /var/lib/jenkins
RUN chown gluon /remoting
# Start-up script to attach the slave to the master
ADD slave.py /var/lib/jenkins/slave.py
USER gluon
WORKDIR /home/jenkins
ENV JENKINS_URL "https://build.ffh.zone/"
ENV JENKINS_SLAVE_ADDRESS ""
ENV SLAVE_EXECUTORS "1"
ENV SLAVE_LABELS "docker"
ENV SLAVE_WORING_DIR ""
ENV CLEAN_WORKING_DIR "true"
CMD [ "python3", "-u", "/var/lib/jenkins/slave.py" ]
# Gluon CI using Jenkins
## Requirements
- Linux system
- with docker installed
- with Hardware Virtualisation (KVM Support)
- Verify using: `lscpu | grep vmx`
- If machine is virtualized host needs to load `kvm_intel` with `nested=1` option and cpuflags need to include `vmx`
## Architecture
![Screenshot from 2019-09-24 00-20-32](https://user-images.githubusercontent.com/601153/65468827-9edf2c80-de65-11e9-9fe0-56c3487719c3.png)
## Installation
You can support the gluon CI with your infrastructure:
1. You need to query @lemoer (freifunk@irrelefant.net) for credentials.
2. He will give you a `SLAVE_NAME` and a `SLAVE_SECRET` for your host.
3. Then go to your docker host and substitute the values for `SLAVE_NAME` and a `SLAVE_SECRET` in the following statements:
``` shell
git clone https://github.com/freifunk-gluon/gluon/
cd gluon/contrib/ci/jenkins-community-slave/
docker build -t gluon-jenkins .
mkdir /var/cache/openwrt_dl_cache/
chown 1000:1000 /var/cache/openwrt_dl_cache
echo "z /dev/kvm 0666 - kvm -" > /etc/tmpfiles.d/kvm.conf
systemd-tmpfiles --create
docker run --detach --restart always \
--env "SLAVE_NAME=whoareyou" \
--env "SLAVE_SECRET=changeme" \
--device /dev/kvm:/dev/kvm \
--volume /var/cache/openwrt_dl_cache/:/dl_cache \
gluon-jenkins
```
4. Check whether the instance is running correctly:
- Your node should appear [here](https://build.ffh.zone/label/gluon-docker/).
- When clicking on it, Jenkins should state "Agent is connected." like here:
![Screenshot from 2019-09-24 01-00-52](https://user-images.githubusercontent.com/601153/65469209-dac6c180-de66-11e9-9d62-0d1c3b6b940b.png)
5. **Your docker container needs to be rebuilt, when the build dependencies of gluon change. As soon as build dependencies have changed, the build dependency api level has to be raised.** After you rebuilt your docker container, notify @lemoer, so he can bump the versioning number.
## Backoff
- If @lemoer is not reachable, please be patient at first if possible. Otherwise contact info@hannover.freifunk.net or join the channel `#freifunkh` on hackint.
from jenkins import Jenkins, JenkinsError, NodeLaunchMethod
import os
import signal
import sys
import subprocess
import shutil
import requests
import time
slave_jar = '/var/lib/jenkins/slave.jar'
slave_name = os.environ['SLAVE_NAME'] if os.environ['SLAVE_NAME'] != '' else 'docker-slave-' + os.environ['HOSTNAME']
jnlp_url = os.environ['JENKINS_URL'] + '/computer/' + slave_name + '/slave-agent.jnlp'
slave_jar_url = os.environ['JENKINS_URL'] + '/jnlpJars/slave.jar'
print(slave_jar_url)
process = None
def clean_dir(dir):
for root, dirs, files in os.walk(dir):
for f in files:
os.unlink(os.path.join(root, f))
for d in dirs:
shutil.rmtree(os.path.join(root, d))
def slave_create(node_name, working_dir, executors, labels):
j = Jenkins(os.environ['JENKINS_URL'], os.environ['JENKINS_USER'], os.environ['JENKINS_PASS'])
j.node_create(node_name, working_dir, num_executors = int(executors), labels = labels, launcher = NodeLaunchMethod.JNLP)
def slave_delete(node_name):
j = Jenkins(os.environ['JENKINS_URL'], os.environ['JENKINS_USER'], os.environ['JENKINS_PASS'])
j.node_delete(node_name)
def slave_download(target):
if os.path.isfile(slave_jar):
os.remove(slave_jar)
r = requests.get(os.environ['JENKINS_URL'] + '/jnlpJars/slave.jar')
with open('/var/lib/jenkins/slave.jar', 'wb') as f:
f.write(r.content)
def slave_run(slave_jar, jnlp_url):
params = [ 'java', '-jar', slave_jar, '-jnlpUrl', jnlp_url ]
if os.environ['JENKINS_SLAVE_ADDRESS'] != '':
params.extend([ '-connectTo', os.environ['JENKINS_SLAVE_ADDRESS' ] ])
if os.environ['SLAVE_SECRET'] == '':
params.extend([ '-jnlpCredentials', os.environ['JENKINS_USER'] + ':' + os.environ['JENKINS_PASS'] ])
else:
params.extend([ '-secret', os.environ['SLAVE_SECRET'] ])
return subprocess.Popen(params, stdout=subprocess.PIPE)
def signal_handler(sig, frame):
if process != None:
process.send_signal(signal.SIGINT)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
def h():
print("ERROR!: please specify environment variables")
print("")
print('docker run -e "SLAVE_NAME=test" -e "SLAVE_SECRET=..." jenkins')
if os.environ.get('SLAVE_NAME') is None:
h()
sys.exit(1)
if os.environ.get('SLAVE_SECRET') is None:
h()
sys.exit(1)
def master_ready(url):
try:
r = requests.head(url, timeout=None)
return r.status_code == requests.codes.ok
except:
return False
while not master_ready(slave_jar_url):
print("Master not ready yet, sleeping for 10sec!")
time.sleep(10)
slave_download(slave_jar)
print('Downloaded Jenkins slave jar.')
if os.environ['SLAVE_WORING_DIR']:
os.setcwd(os.environ['SLAVE_WORING_DIR'])
if os.environ['CLEAN_WORKING_DIR'] == 'true':
clean_dir(os.getcwd())
print("Cleaned up working directory.")
if os.environ['SLAVE_NAME'] == '':
slave_create(slave_name, os.getcwd(), os.environ['SLAVE_EXECUTORS'], os.environ['SLAVE_LABELS'])
print('Created temporary Jenkins slave.')
process = slave_run(slave_jar, jnlp_url)
print('Started Jenkins slave with name "' + slave_name + '" and labels [' + os.environ['SLAVE_LABELS'] + '].')
process.wait()
print('Jenkins slave stopped.')
if os.environ['SLAVE_NAME'] == '':
slave_delete(slave_name)
print('Removed temporary Jenkins slave.')
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment