[Bf-blender-cvs] [19b597c55d5] master: Tests: improve finding of Blender executables in benchmarking
Brecht Van Lommel
noreply at git.blender.org
Thu Jul 22 16:38:27 CEST 2021
Commit: 19b597c55d5d6cd6eb6cecdcff6db3e3fb5525ef
Author: Brecht Van Lommel
Date: Tue Jul 20 16:52:22 2021 +0200
Branches: master
https://developer.blender.org/rB19b597c55d5d6cd6eb6cecdcff6db3e3fb5525ef
Tests: improve finding of Blender executables in benchmarking
* Allow specifying a folder and automatically setting the proper executable
name depending on the operating system
* Use executables from configs for listing devices instead of a blender
command being available
===================================================================
M tests/performance/api/config.py
M tests/performance/api/environment.py
M tests/performance/benchmark
===================================================================
diff --git a/tests/performance/api/config.py b/tests/performance/api/config.py
index 68f4df8d487..283d1ff16ec 100644
--- a/tests/performance/api/config.py
+++ b/tests/performance/api/config.py
@@ -112,7 +112,7 @@ class TestConfig:
self.base_dir = env.base_dir / name
self.logs_dir = self.base_dir / 'logs'
- config = self._read_config_module()
+ config = TestConfig._read_config_module(self.base_dir)
self.tests = TestCollection(env,
getattr(config, 'tests', ['*']),
getattr(config, 'categories', ['*']))
@@ -154,10 +154,17 @@ class TestConfig:
with open(config_file, 'w') as f:
f.write(default_config)
- def _read_config_module(self) -> None:
+ @staticmethod
+ def read_blender_executables(env, name) -> List:
+ config = TestConfig._read_config_module(env.base_dir / name)
+ builds = getattr(config, 'builds', {})
+ return [pathlib.Path(build) for build in builds.values()]
+
+ @staticmethod
+ def _read_config_module(base_dir: pathlib.Path) -> None:
# Import config.py as a module.
import importlib.util
- spec = importlib.util.spec_from_file_location("testconfig", self.base_dir / 'config.py')
+ spec = importlib.util.spec_from_file_location("testconfig", base_dir / 'config.py')
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
@@ -195,14 +202,14 @@ class TestConfig:
# Get entries for revisions based on existing builds.
for revision_name, executable in self.builds.items():
- executable_path = pathlib.Path(executable)
- if not executable_path.exists():
+ executable_path = env._blender_executable_from_path(pathlib.Path(executable))
+ if not executable_path:
sys.stderr.write(f'Error: build {executable} not found\n')
sys.exit(1)
env.set_blender_executable(executable_path)
git_hash, _ = env.run_in_blender(get_build_hash, {})
- env.unset_blender_executable()
+ env.set_default_blender_executable()
mtime = executable_path.stat().st_mtime
entries += self._get_entries(revision_name, git_hash, executable, mtime)
diff --git a/tests/performance/api/environment.py b/tests/performance/api/environment.py
index c9ddd493394..1a8f5ceab51 100644
--- a/tests/performance/api/environment.py
+++ b/tests/performance/api/environment.py
@@ -27,9 +27,10 @@ class TestEnvironment:
self.git_executable = 'git'
self.cmake_executable = 'cmake'
self.cmake_options = ['-DWITH_INTERNATIONAL=OFF', '-DWITH_BUILDINFO=OFF']
- self.unset_blender_executable()
self.log_file = None
self.machine = None
+ self._init_default_blender_executable()
+ self.set_default_blender_executable()
def get_machine(self, need_gpus: bool=True) -> None:
if not self.machine or (need_gpus and not self.machine.has_gpus):
@@ -46,7 +47,7 @@ class TestEnvironment:
print(f'Init {self.base_dir}')
self.base_dir.mkdir(parents=True, exist_ok=True)
- if len(self.get_configs(names_only=True)) == 0:
+ if len(self.get_configs_names()) == 0:
config_dir = self.base_dir / 'default'
print(f'Creating default configuration in {config_dir}')
TestConfig.write_default_config(self, config_dir)
@@ -77,7 +78,7 @@ class TestEnvironment:
print('Done')
- def checkout(self) -> None:
+ def checkout(self, git_hash) -> None:
# Checkout Blender revision
if not self.blender_dir.exists():
sys.stderr.write('\n\nError: no build set up, run `./benchmark init --build` first\n')
@@ -87,8 +88,6 @@ class TestEnvironment:
self.call([self.git_executable, 'reset', '--hard', 'HEAD'], self.blender_dir)
self.call([self.git_executable, 'checkout', '--detach', git_hash], self.blender_dir)
- self.build()
-
def build(self) -> None:
# Build Blender revision
if not self.build_dir.exists():
@@ -98,21 +97,54 @@ class TestEnvironment:
jobs = str(multiprocessing.cpu_count())
self.call([self.cmake_executable, '.'] + self.cmake_options, self.build_dir)
self.call([self.cmake_executable, '--build', '.', '-j', jobs, '--target', 'install'], self.build_dir)
+ self._init_default_blender_executable()
def set_blender_executable(self, executable_path: pathlib.Path) -> None:
# Run all Blender commands with this executable.
self.blender_executable = executable_path
- def unset_blender_executable(self) -> None:
+ def _blender_executable_name(self) -> pathlib.Path:
if platform.system() == "Windows":
- self.blender_executable = self.build_dir / 'bin' / 'blender.exe'
+ return pathlib.Path('blender.exe')
elif platform.system() == "Darwin":
- self.blender_executable = self.build_dir / 'bin' / 'Blender.app' / 'Contents' / 'MacOS' / 'Blender'
+ return pathlib.Path('Blender.app') / 'Contents' / 'MacOS' / 'Blender'
else:
- self.blender_executable = self.build_dir / 'bin' / 'blender'
-
- if not self.blender_executable.exists():
- self.blender_executable = 'blender'
+ return pathlib.Path('blender')
+
+ def _blender_executable_from_path(self, executable: pathlib.Path) -> pathlib.Path:
+ if executable.is_dir():
+ # Directory
+ executable = executable / self._blender_executable_name()
+ elif not executable.is_file() and executable.name == 'blender':
+ # Executable path without proper path on Windows or macOS.
+ executable = executable.parent() / self._blender_executable_name()
+
+ if executable.is_file():
+ return executable
+
+ return None
+
+ def _init_default_blender_executable(self) -> None:
+ # Find a default executable to run commands independent of testing a specific build.
+ # Try own built executable.
+ built_executable = self._blender_executable_from_path(self.build_dir / 'bin')
+ if built_executable:
+ self.default_blender_executable = built_executable
+ return
+
+ # Try find an executable in the configs.
+ for config_name in self.get_config_names():
+ for executable in TestConfig.read_blender_executables(self, config_name):
+ executable = self._blender_executable_from_path(executable)
+ if executable:
+ self.default_blender_executable = executable
+ return
+
+ # Fallback to a "blender" command in the hope it's available.
+ self.default_blender_executable = pathlib.Path("blender")
+
+ def set_default_blender_executable(self) -> None:
+ self.blender_executable = self.default_blender_executable
def set_log_file(self, filepath: pathlib.Path, clear=True) -> None:
# Log all commands and output to this file.
@@ -219,19 +251,27 @@ class TestEnvironment:
filepaths.append(pathlib.Path(filename))
return filepaths
+ def get_config_names(self) -> List:
+ names = []
+
+ if self.base_dir.exists():
+ for dirname in os.listdir(self.base_dir):
+ dirpath = self.base_dir / dirname / 'config.py'
+ if dirpath.exists():
+ names.append(dirname)
+
+ return names
+
def get_configs(self, name: str=None, names_only: bool=False) -> List:
# Get list of configurations in the benchmarks directory.
configs = []
- if self.base_dir.exists():
- for dirname in os.listdir(self.base_dir):
- if not name or dirname == name:
- dirpath = self.base_dir / dirname / 'config.py'
- if dirpath.exists():
- if names_only:
- configs.append(dirname)
- else:
- configs.append(TestConfig(self, dirname))
+ for config_name in self.get_config_names():
+ if not name or config_name == name:
+ if names_only:
+ configs.append(config_name)
+ else:
+ configs.append(TestConfig(self, config_name))
return configs
diff --git a/tests/performance/benchmark b/tests/performance/benchmark
index 3b43bd0aa96..4ca1d0eda4c 100755
--- a/tests/performance/benchmark
+++ b/tests/performance/benchmark
@@ -112,7 +112,7 @@ def run_entry(env: api.TestEnvironment, config: api.TestConfig, row: List, entry
env.set_blender_executable(pathlib.Path(entry.executable))
else:
env.checkout(git_hash)
- env.build(git_hash)
+ env.build()
# Run test and update output and status.
entry.status = 'running'
@@ -126,7 +126,7 @@ def run_entry(env: api.TestEnvironment, config: api.TestConfig, row: List, entry
# Restore default logging and Blender executable.
env.unset_log_file()
- env.unset_blender_executable()
+ env.set_default_blender_executable()
return True
@@ -155,7 +155,7 @@ def cmd_list(env: api.TestEnvironment, argv: List) -> None:
print('')
print('CONFIGS')
- configs = env.get_configs(names_only=True)
+ configs = env.get_config_names()
for config_name in configs:
print(config_name)
More information about the Bf-blender-cvs
mailing list