[Bf-blender-cvs] [283d96de117] master: Buildbot: refactor all build slave code
Brecht Van Lommel
noreply at git.blender.org
Mon Sep 2 17:38:07 CEST 2019
Commit: 283d96de1170f7c42a43bde4e30fdb438939978f
Author: Brecht Van Lommel
Date: Sat Aug 31 12:13:29 2019 +0200
Branches: master
https://developer.blender.org/rB283d96de1170f7c42a43bde4e30fdb438939978f
Buildbot: refactor all build slave code
* Move common code into buildbot_utils.py
* Remove legacy code from removed builders
* Split code into smaller functions
Differential Revision: https://developer.blender.org/D5642
===================================================================
A build_files/buildbot/buildbot_utils.py
M build_files/buildbot/slave_compile.py
M build_files/buildbot/slave_pack.py
M build_files/buildbot/slave_rsync.py
M build_files/buildbot/slave_test.py
M build_files/buildbot/slave_update.py
===================================================================
diff --git a/build_files/buildbot/buildbot_utils.py b/build_files/buildbot/buildbot_utils.py
new file mode 100644
index 00000000000..653f8ac2d97
--- /dev/null
+++ b/build_files/buildbot/buildbot_utils.py
@@ -0,0 +1,92 @@
+# ##### BEGIN GPL LICENSE BLOCK #####
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ##### END GPL LICENSE BLOCK #####
+
+# <pep8 compliant>
+
+import argparse
+import os
+import subprocess
+import sys
+
+class Builder:
+ def __init__(self, name, branch):
+ self.name = name
+ self.branch = branch
+
+ # Buildbot runs from build/ directory
+ self.blender_dir = os.path.abspath(os.path.join('..', 'blender.git'))
+ self.build_dir = os.path.abspath(os.path.join('..', 'build', name))
+ self.install_dir = os.path.abspath(os.path.join('..', 'install', name))
+ self.upload_dir = os.path.abspath(os.path.join('..', 'install'))
+
+ # Detect platform
+ if name.startswith('mac'):
+ self.platform = 'mac'
+ self.command_prefix = []
+ elif name.startswith('linux'):
+ self.platform = 'linux'
+ self.command_prefix = ['scl', 'enable', 'devtoolset-6', '--']
+ elif name.startswith('win'):
+ self.platform = 'win'
+ self.command_prefix = []
+ else:
+ raise ValueError('Unkonw platform for builder ' + self.platform)
+
+ # Always 64 bit now
+ self.bits = 64
+
+def create_builder_from_arguments():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('builder_name')
+ parser.add_argument('branch', default='master', nargs='?')
+ args = parser.parse_args()
+ return Builder(args.builder_name, args.branch)
+
+
+class VersionInfo:
+ def __init__(self, builder):
+ # Get version information
+ buildinfo_h = os.path.join(builder.build_dir, "source", "creator", "buildinfo.h")
+ blender_h = os.path.join(builder.blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h")
+
+ version_number = int(self._parse_header_file(blender_h, 'BLENDER_VERSION'))
+ self.version = "%d.%d" % (version_number // 100, version_number % 100)
+ self.hash = self._parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1]
+
+ def _parse_header_file(self, filename, define):
+ import re
+ regex = re.compile("^#\s*define\s+%s\s+(.*)" % define)
+ with open(filename, "r") as file:
+ for l in file:
+ match = regex.match(l)
+ if match:
+ return match.group(1)
+ return None
+
+
+def call(cmd, env=None, exit_on_error=True):
+ print(' '.join(cmd))
+
+ # Flush to ensure correct order output on Windows.
+ sys.stdout.flush()
+ sys.stderr.flush()
+
+ retcode = subprocess.call(cmd, env=env)
+ if exit_on_error and retcode != 0:
+ sys.exit(retcode)
+ return retcode
diff --git a/build_files/buildbot/slave_compile.py b/build_files/buildbot/slave_compile.py
index 55543055e5b..84667e663f6 100644
--- a/build_files/buildbot/slave_compile.py
+++ b/build_files/buildbot/slave_compile.py
@@ -18,149 +18,80 @@
# <pep8 compliant>
+import buildbot_utils
import os
-import subprocess
-import sys
import shutil
-# get builder name
-if len(sys.argv) < 2:
- sys.stderr.write("Not enough arguments, expecting builder name\n")
- sys.exit(1)
-
-builder = sys.argv[1]
-
-# we run from build/ directory
-blender_dir = os.path.join('..', 'blender.git')
-
-
-def parse_header_file(filename, define):
- import re
- regex = re.compile("^#\s*define\s+%s\s+(.*)" % define)
- with open(filename, "r") as file:
- for l in file:
- match = regex.match(l)
- if match:
- return match.group(1)
- return None
-
-if 'cmake' in builder:
- # cmake
-
- # Some fine-tuning configuration
- blender_dir = os.path.abspath(blender_dir)
- build_dir = os.path.abspath(os.path.join('..', 'build', builder))
- install_dir = os.path.abspath(os.path.join('..', 'install', builder))
- targets = ['blender']
- command_prefix = []
-
- bits = 64
-
- # Config file to be used (relative to blender's sources root)
- cmake_config_file = "build_files/cmake/config/blender_release.cmake"
-
- # Set build options.
- cmake_options = []
- cmake_extra_options = ['-DCMAKE_BUILD_TYPE:STRING=Release',
- '-DWITH_GTESTS=ON']
-
- if builder.startswith('mac'):
- # Set up OSX architecture
- if builder.endswith('x86_64_10_9_cmake'):
- cmake_extra_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64')
- cmake_extra_options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9')
-
- elif builder.startswith('win'):
- if builder.startswith('win64'):
- cmake_options.extend(['-G', 'Visual Studio 15 2017 Win64'])
- elif builder.startswith('win32'):
- bits = 32
- cmake_options.extend(['-G', 'Visual Studio 15 2017'])
-
- elif builder.startswith('linux'):
- cmake_config_file = "build_files/buildbot/config/blender_linux.cmake"
- tokens = builder.split("_")
- glibc = tokens[1]
- if glibc == 'glibc224':
- deb_name = "stretch"
- if builder.endswith('x86_64_cmake'):
- chroot_name = 'buildbot_' + deb_name + '_x86_64'
- elif builder.endswith('i686_cmake'):
- bits = 32
- chroot_name = 'buildbot_' + deb_name + '_i686'
- command_prefix = ['schroot', '-c', chroot_name, '--']
- elif glibc == 'glibc217':
- command_prefix = ['scl', 'enable', 'devtoolset-6', '--']
-
- cmake_options.append("-C" + os.path.join(blender_dir, cmake_config_file))
-
- # Prepare CMake options needed to configure cuda binaries compilation, 64bit only.
- if bits == 64:
- cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=ON")
- cmake_options.append("-DCUDA_64_BIT_DEVICE_CODE=ON")
- else:
- cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=OFF")
+def get_cmake_options(builder):
+ config_file = "build_files/cmake/config/blender_release.cmake"
+ options = ['-DCMAKE_BUILD_TYPE:STRING=Release', '-DWITH_GTESTS=ON']
+
+ if builder.platform == 'mac':
+ options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64')
+ options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9')
+ elif builder.platform == 'win':
+ options.extend(['-G', 'Visual Studio 15 2017 Win64'])
+ elif builder.platform == 'linux':
+ config_file = "build_files/buildbot/config/blender_linux.cmake"
+
+ options.append("-C" + os.path.join(builder.blender_dir, config_file))
+ options.append("-DCMAKE_INSTALL_PREFIX=%s" % (builder.install_dir))
+
+ return options
- cmake_options.append("-DCMAKE_INSTALL_PREFIX=%s" % (install_dir))
+def update_git(builder):
+ # Do extra git fetch because not all platform/git/buildbot combinations
+ # update the origin remote, causing buildinfo to detect local changes.
+ os.chdir(builder.blender_dir)
- cmake_options += cmake_extra_options
+ print("Fetching remotes")
+ command = ['git', 'fetch', '--all']
+ buildbot_utils.call(builder.command_prefix + command)
+def clean_directories(builder):
# Make sure no garbage remained from the previous run
- if os.path.isdir(install_dir):
- shutil.rmtree(install_dir)
-
- for target in targets:
- print("Building target %s" % (target))
- # Construct build directory name based on the target
- target_build_dir = build_dir
- target_command_prefix = command_prefix[:]
- if target != 'blender':
- target_build_dir += '_' + target
- target_name = 'install'
- # Tweaking CMake options to respect the target
- target_cmake_options = cmake_options[:]
- # Do extra git fetch because not all platform/git/buildbot combinations
- # update the origin remote, causing buildinfo to detect local changes.
- os.chdir(blender_dir)
- print("Fetching remotes")
- command = ['git', 'fetch', '--all']
- print(command)
- retcode = subprocess.call(target_command_prefix + command)
- if retcode != 0:
- sys.exit(retcode)
- # Make sure build directory exists and enter it
- if not os.path.isdir(target_build_dir):
- os.mkdir(target_build_dir)
- os.chdir(target_build_dir)
- # Configure the build
- print("CMake options:")
- print(target_cmake_options)
- if os.path.exists('CMakeCache.txt'):
- print("Removing CMake cache")
- os.remove('CMakeCache.txt')
- # Remove buildinfo files to force buildbot to re-generate them.
- for buildinfo in ('buildinfo.h', 'buildinfo.h.txt', ):
- full_path = os.path.join('source', 'creator', buildinfo)
- if os.path.exists(full_path):
- print("Removing {}" . format(buildinfo))
- os.remove(full_path)
- retcode = subprocess.call(target_command_prefix + ['cmake', blender_dir] + target_cmake_options)
- if retcode != 0:
- print('Configuration FAILED!')
- sys.exit(retcode)
-
- if 'win32' in builder or 'win64' in builder:
- command = ['cmake', '--build', '.', '--target', target_name, '--config', 'Release']
- else:
- command = ['make', '-s', '-j2', target_name]
-
- print("Executing command:")
-
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list