[Bf-blender-cvs] [2cbc328bdcf] buildbot-lts: Build bot release pipeline
Jeroen Bakker
noreply at git.blender.org
Fri Jul 3 16:23:43 CEST 2020
Commit: 2cbc328bdcf7a48243344b8b33914a43b497df23
Author: Jeroen Bakker
Date: Fri Jul 3 16:19:50 2020 +0200
Branches: buildbot-lts
https://developer.blender.org/rB2cbc328bdcf7a48243344b8b33914a43b497df23
Build bot release pipeline
* downloads builds from builder.blender.org
* generates checksums (still need to cleanup filenames)
* create source archive
* upload to download.blender.org
Next steps
* Next step is to place them in the correct folder
* file permissions download.blender.org
* snap package
* steam packages
* release notes
TODO:
* add more generic functions in master.cfg
* use a separate folder per build
===================================================================
M build_files/buildbot-lts/README.md
A build_files/buildbot-lts/docker/Dockerfile
A build_files/buildbot-lts/docker/bin/create_checksum.sh
A build_files/buildbot-lts/docker/bin/upload_file.sh
A build_files/buildbot-lts/docker/buildbot.tac
A build_files/buildbot-lts/docker/docker-compose.yml
A build_files/buildbot-lts/docker/docker-entrypoint.sh
A build_files/buildbot-lts/docker/master.cfg
A build_files/buildbot-lts/docker/worker/Dockerfile
A build_files/buildbot-lts/docker/worker/buildbot.tac
===================================================================
diff --git a/build_files/buildbot-lts/README.md b/build_files/buildbot-lts/README.md
index 73750be84f6..ec2a96c4652 100644
--- a/build_files/buildbot-lts/README.md
+++ b/build_files/buildbot-lts/README.md
@@ -13,3 +13,22 @@ private keys and the process needs to be controlled security wise.
But of course the source and configurations are public available for anyone to
check, develop and use.
+
+Setting up build-bot
+--------------------
+
+instructions from https://github.com/cjolowicz/docker-buildbot.
+
+Create custom buildbot worker containing packages we need for building (git, wget).
+
+ cd docker
+ docker build -t buildbot --no-cache .
+ cd worker
+ docker build -t buildbot-worker --no-cache .
+
+ docker network create net-buildbot
+
+ docker rm lts-buildbot && docker run --init --name lts-buildbot --network net-buildbot --publish=127.0.0.1:8010:8010 -t -i buildbot
+
+ docker rm lts-worker && docker run --init --name lts-worker --network net-buildbot --name lts-worker -e BUILDMASTER=lts-buildbot -e WORKERNAME=lts-worker -e WORKERPASS=secret -t -i buildbot-worker
+
diff --git a/build_files/buildbot-lts/docker/Dockerfile b/build_files/buildbot-lts/docker/Dockerfile
new file mode 100644
index 00000000000..bf433f1afee
--- /dev/null
+++ b/build_files/buildbot-lts/docker/Dockerfile
@@ -0,0 +1,36 @@
+FROM ubuntu:18.04
+
+RUN apt update && apt upgrade -y
+RUN apt install -y \
+ gosu \
+ wget \
+ openssh-client \
+ build-essential \
+ libffi-dev \
+ libssl-dev \
+ python3-dev \
+ python3-pip
+
+
+ENV BUILDBOT_VERSION 2.5.0
+RUN pip3 --no-cache-dir install --upgrade pip && pip --no-cache-dir install \
+ buildbot[bundle,tls]==$BUILDBOT_VERSION \
+ buildbot-docker-swarm-worker
+
+COPY buildbot.tac /var/lib/buildbot/
+COPY docker-entrypoint.sh /usr/local/bin/
+COPY master.cfg /etc/buildbot/
+COPY bin/create_checksum.sh /var/lib/buildbot/bin/
+COPY bin/upload_file.sh /var/lib/buildbot/bin/
+COPY .ssh/id_rsa /var/lib/buildbot/.ssh/
+
+RUN adduser --home /var/lib/buildbot --disabled-password --gecos '' buildbot
+WORKDIR /var/lib/buildbot
+RUN ln -s /etc/buildbot/master.cfg
+
+VOLUME /var/lib/buildbot
+EXPOSE 8010
+EXPOSE 9989
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD ["twistd", "--pidfile=", "--nodaemon", "--python=buildbot.tac"]
\ No newline at end of file
diff --git a/build_files/buildbot-lts/docker/bin/create_checksum.sh b/build_files/buildbot-lts/docker/bin/create_checksum.sh
new file mode 100755
index 00000000000..fe69e43e5db
--- /dev/null
+++ b/build_files/buildbot-lts/docker/bin/create_checksum.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+set -e
+md5sum $1 > $2.md5
+sha256sum $1 > $2.sha256
\ No newline at end of file
diff --git a/build_files/buildbot-lts/docker/bin/upload_file.sh b/build_files/buildbot-lts/docker/bin/upload_file.sh
new file mode 100755
index 00000000000..95fb71388ec
--- /dev/null
+++ b/build_files/buildbot-lts/docker/bin/upload_file.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+set -e
+if [ ! -f /var/lib/buildbot/.ssh/known_hosts ]
+then
+ ssh-keyscan download.blender.org > /var/lib/buildbot/.ssh/known_hosts
+fi
+scp $1 jeroen at download.blender.org:$2
\ No newline at end of file
diff --git a/build_files/buildbot-lts/docker/buildbot.tac b/build_files/buildbot-lts/docker/buildbot.tac
new file mode 100644
index 00000000000..930d392cc50
--- /dev/null
+++ b/build_files/buildbot-lts/docker/buildbot.tac
@@ -0,0 +1,20 @@
+import os
+import sys
+
+from twisted.application import service
+from twisted.python.log import FileLogObserver
+from twisted.python.log import ILogObserver
+
+from buildbot.master import BuildMaster
+
+basedir = os.environ.get("BUILDBOT_BASEDIR",
+ os.path.abspath(os.path.dirname(__file__)))
+configfile = 'master.cfg'
+
+# note: this line is matched against to check that this is a buildmaster
+# directory; do not edit it.
+application = service.Application('buildmaster')
+application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit)
+
+m = BuildMaster(basedir, configfile, umask=None)
+m.setServiceParent(application)
\ No newline at end of file
diff --git a/build_files/buildbot-lts/docker/docker-compose.yml b/build_files/buildbot-lts/docker/docker-compose.yml
new file mode 100644
index 00000000000..cf0db3f16c4
--- /dev/null
+++ b/build_files/buildbot-lts/docker/docker-compose.yml
@@ -0,0 +1,36 @@
+version: '1'
+services:
+ buildbot:
+ image: buildbot/buildbot-master:master
+ env_file:
+ - db.env
+ environment:
+ - BUILDBOT_CONFIG_DIR=config
+ - BUILDBOT_CONFIG_URL=https://github.com/buildbot/buildbot-docker-example-config/archive/master.tar.gz
+ - BUILDBOT_WORKER_PORT=9989
+ - BUILDBOT_WEB_URL=http://localhost:8010/
+ - BUILDBOT_WEB_PORT=tcp:port=8010
+ links:
+ - db
+ depends_on:
+ - db
+ ports:
+ - "8010:8010"
+ db:
+ env_file:
+ - db.env
+ image: "postgres:9.4"
+ expose:
+ - 5432
+
+ worker:
+ image: "buildbot/buildbot-worker:master"
+ environment:
+ BUILDMASTER: buildbot
+ BUILDMASTER_PORT: 9989
+ WORKERNAME: lts-worker
+ WORKERPASS: pass
+ WORKER_ENVIRONMENT_BLACKLIST: DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST
+
+ links:
+ - buildbot
\ No newline at end of file
diff --git a/build_files/buildbot-lts/docker/docker-entrypoint.sh b/build_files/buildbot-lts/docker/docker-entrypoint.sh
new file mode 100755
index 00000000000..b7298560662
--- /dev/null
+++ b/build_files/buildbot-lts/docker/docker-entrypoint.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+set -e
+
+buildbot upgrade-master .
+
+chown buildbot /var/lib/buildbot/
+chown buildbot /var/lib/buildbot/.ssh/
+chown -R buildbot /var/lib/buildbot/*
+
+if [ -S /var/run/docker.sock ]
+then
+ group=$(stat -c '%g' /var/run/docker.sock)
+else
+ group=buildbot
+fi
+
+gosu buildbot:$group "$@"
\ No newline at end of file
diff --git a/build_files/buildbot-lts/docker/master.cfg b/build_files/buildbot-lts/docker/master.cfg
new file mode 100644
index 00000000000..1dfa8bbdf7c
--- /dev/null
+++ b/build_files/buildbot-lts/docker/master.cfg
@@ -0,0 +1,240 @@
+# -*- python -*-
+# ex: set filetype=python:
+
+import os
+
+from buildbot.plugins import *
+
+# This is a sample buildmaster config file. It must be installed as
+# 'master.cfg' in your buildmaster's base directory.
+
+# This is the dictionary that the buildmaster pays attention to. We also use
+# a shorter alias to save typing.
+c = BuildmasterConfig = {}
+
+####### WORKERS
+
+# The 'workers' list defines the set of recognized workers. Each element is
+# a Worker object, specifying a unique worker name and password. The same
+# worker name and password must be configured on the worker.
+
+c['workers'] = [worker.Worker("lts-worker", 'secret')]
+
+if 'BUILDBOT_MQ_URL' in os.environ:
+ c['mq'] = {
+ 'type' : 'wamp',
+ 'router_url': os.environ['BUILDBOT_MQ_URL'],
+ 'realm': os.environ.get('BUILDBOT_MQ_REALM', 'buildbot').decode('utf-8'),
+ 'debug' : 'BUILDBOT_MQ_DEBUG' in os.environ,
+ 'debug_websockets' : 'BUILDBOT_MQ_DEBUG' in os.environ,
+ 'debug_lowlevel' : 'BUILDBOT_MQ_DEBUG' in os.environ,
+ }
+# 'protocols' contains information about protocols which master will use for
+# communicating with workers. You must define at least 'port' option that workers
+# could connect to your master with this protocol.
+# 'port' must match the value configured into the workers (with their
+# --master option)
+c['protocols'] = {'pb': {'port': os.environ.get("BUILDBOT_WORKER_PORT", 9989)}}
+
+####### CHANGESOURCES
+
+# the 'change_source' setting tells the buildmaster how it should find out
+# about source code changes. We don't build when sources change.
+c['change_source'] = []
+
+####### SCHEDULERS
+
+# Configure the Schedulers, which decide how to react to incoming changes. In this
+# case, just kick off a 'runtests' build
+
+c['schedulers'] = [
+ schedulers.ForceScheduler(
+ name="build",
+ builderNames=["release-blender-283"],
+ properties=[
+ util.StringParameter(
+ name="blender_version",
+ label="Version of blender.",
+ default="2.83"),
+ util.StringParameter(
+ name="blender_version_full",
+ label="Version of blender (Full).",
+ default="2.83.1")
+ ]
+ ),
+
+ schedulers.Triggerable(
+ name="deploy-source-archive",
+ builderNames=["deploy-source-archive"]
+ ),
+
+ schedulers.Triggerable(
+ name="deploy-buildbot-packages",
+ builderNames=["deploy-buildbot-packages"]
+ ),
+]
+
+####### BUILDERS
+
+# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
+# what steps, and which workers can execute them. Note that any particular build will
+# only take place on one worker.
+
+deploy_source_archive_factory = util.BuildFactory()
+deploy_source_archive_factory.addStep(steps.Git(name="Checkout Blender Source", repourl='https://git.blender.org/blender.git', branch=util.Interpolate('blender-v%(prop:blender_version)s-release'), mode='full'))
+deploy_source_archive_factory.addStep(steps.ShellSequence(name="Make source archive", commands=[
+ util.ShellArg(command=["make", "source_archive"], logfile='make_source_archive'),
+ # The next two shell commands are only needed during development
+ util.ShellArg(command=["mv", "blender-2.83.2-beta.tar.xz", util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz')]),
+ util.ShellArg(command=["mv", "blender-2.83.2-beta.tar.xz.md5sum", util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz.md5sum')]),
+]))
+deploy_source_archive_factory.addStep(steps.FileUpload(
+ name="Upload source archive",
+ workersrc=util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz'),
+ masterdest=util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-v%(prop:blender_version_full)s.tar.xz')))
+deploy_source_archive_factory.addStep(steps.FileUpload(
+ name="Upload source archive checksum",
+ workersrc=util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz.md5sum'),
+ masterd
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list