[Bf-blender-cvs] [93901e7f0a0] master: Tests: speed up render tests by running multiple in the same process
Brecht Van Lommel
noreply at git.blender.org
Sat May 11 00:13:09 CEST 2019
Commit: 93901e7f0a05ba471f3b4c0201500d9dfcd68c2c
Author: Brecht Van Lommel
Date: Fri May 10 23:00:35 2019 +0200
Branches: master
https://developer.blender.org/rB93901e7f0a05ba471f3b4c0201500d9dfcd68c2c
Tests: speed up render tests by running multiple in the same process
Blender startup time and shader compilation is a big factor when running
hundreds of tests, so now all renders in the same ctest run in the same
process. If a test crashes, the remaining tests in the same category will
be marked as skipped.
Benchmarked on a quad core with ctest -j8.
cycles: 118.1s -> 94.3s
eevee: 66.2s -> 29.2s
workbench: 31.7s -> 8.6s
===================================================================
M tests/python/cycles_render_tests.py
M tests/python/eevee_render_tests.py
M tests/python/modules/render_report.py
M tests/python/opengl_draw_tests.py
M tests/python/workbench_render_tests.py
===================================================================
diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py
index 36f5459c2f7..a66409d85c3 100755
--- a/tests/python/cycles_render_tests.py
+++ b/tests/python/cycles_render_tests.py
@@ -9,77 +9,76 @@ import subprocess
import sys
-def render_file(filepath, output_filepath):
- dirname = os.path.dirname(filepath)
- basedir = os.path.dirname(dirname)
- subject = os.path.basename(dirname)
-
- frame_filepath = output_filepath + '0001.png'
-
- common_args = [
- "-noaudio",
- "--factory-startup",
- "--enable-autoexec",
- filepath,
- "-E", "CYCLES",
- "-o", output_filepath,
- "-F", "PNG"]
+def render_files(filepaths, output_filepaths):
+ command = [BLENDER, "--background"]
# OSL and GPU examples
# custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"]
# custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"]
custom_args = os.getenv('CYCLESTEST_ARGS')
custom_args = shlex.split(custom_args) if custom_args else []
- common_args += custom_args
-
- if subject == 'opengl':
- command = [BLENDER, "--window-geometry", "0", "0", "1", "1"]
- command += common_args
- command += ['--python', os.path.join(basedir, "util", "render_opengl.py")]
- elif subject == 'bake':
- command = [BLENDER, "--background"]
- command += common_args
- command += ['--python', os.path.join(basedir, "util", "render_bake.py")]
- elif subject == 'denoise_animation':
- command = [BLENDER, "--background"]
- command += common_args
- command += ['--python', os.path.join(basedir, "util", "render_denoise.py")]
- else:
- command = [BLENDER, "--background"]
- command += common_args
- command += ["-f", "1"]
+ for filepath, output_filepath in zip(filepaths, output_filepaths):
+ dirname = os.path.dirname(filepath)
+ basedir = os.path.dirname(dirname)
+ subject = os.path.basename(dirname)
+
+ frame_filepath = output_filepath + '0001.png'
+
+ common_args = [
+ "-noaudio",
+ "--factory-startup",
+ "--enable-autoexec",
+ filepath,
+ "-E", "CYCLES",
+ "-o", output_filepath,
+ "-F", "PNG"]
+
+ common_args += custom_args
+
+ if subject == 'bake':
+ command.extend(common_args)
+ command.extend(['--python', os.path.join(basedir, "util", "render_bake.py")])
+ elif subject == 'denoise_animation':
+ command.extend(common_args)
+ command.extend(['--python', os.path.join(basedir, "util", "render_denoise.py")])
+ else:
+ command.extend(common_args)
+ command.extend(["-f", "1"])
+
+ error = None
try:
# Success
output = subprocess.check_output(command)
- if os.path.exists(frame_filepath):
- shutil.copy(frame_filepath, output_filepath)
- os.remove(frame_filepath)
if VERBOSE:
print(" ".join(command))
print(output.decode("utf-8"))
- return None
except subprocess.CalledProcessError as e:
# Error
- if os.path.exists(frame_filepath):
- os.remove(frame_filepath)
if VERBOSE:
print(" ".join(command))
print(e.output.decode("utf-8"))
- if b"Error: engine not found" in e.output:
- return "NO_ENGINE"
- elif b"blender probably wont start" in e.output:
- return "NO_START"
- return "CRASH"
+ error = "CRASH"
except BaseException as e:
# Crash
- if os.path.exists(frame_filepath):
- os.remove(frame_filepath)
if VERBOSE:
print(" ".join(command))
- print(e)
- return "CRASH"
+ print(e.decode("utf-8"))
+ error = "CRASH"
+
+ # Detect missing filepaths and consider those errors
+ errors = []
+ for output_filepath in output_filepaths:
+ frame_filepath = output_filepath + '0001.png'
+ if os.path.exists(frame_filepath):
+ shutil.copy(frame_filepath, output_filepath)
+ os.remove(frame_filepath)
+ errors.append(None)
+ else:
+ errors.append(error)
+ error = 'SKIPPED'
+ return errors
def create_argparse():
parser = argparse.ArgumentParser()
@@ -108,7 +107,7 @@ def main():
report.set_pixelated(True)
report.set_reference_dir("cycles_renders")
report.set_compare_engines('cycles', 'eevee')
- ok = report.run(test_dir, render_file)
+ ok = report.run(test_dir, render_files)
sys.exit(not ok)
diff --git a/tests/python/eevee_render_tests.py b/tests/python/eevee_render_tests.py
index c0536e05164..d2c37dbcb2e 100755
--- a/tests/python/eevee_render_tests.py
+++ b/tests/python/eevee_render_tests.py
@@ -49,57 +49,61 @@ if inside_blender:
sys.exit(1)
-def render_file(filepath, output_filepath):
- dirname = os.path.dirname(filepath)
- basedir = os.path.dirname(dirname)
- subject = os.path.basename(dirname)
-
- frame_filepath = output_filepath + '0001.png'
-
+def render_files(filepaths, output_filepaths):
command = [
BLENDER,
"--background",
"-noaudio",
"--factory-startup",
- "--enable-autoexec",
- filepath,
- "-E", "BLENDER_EEVEE",
- "-P",
- os.path.realpath(__file__),
- "-o", output_filepath,
- "-F", "PNG",
- "-f", "1"]
+ "--enable-autoexec"]
+
+ for filepath, output_filepath in zip(filepaths, output_filepaths):
+ frame_filepath = output_filepath + '0001.png'
+ if os.path.exists(frame_filepath):
+ os.remove(frame_filepath)
+ command.extend([
+ filepath,
+ "-E", "BLENDER_EEVEE",
+ "-P",
+ os.path.realpath(__file__),
+ "-o", output_filepath,
+ "-F", "PNG",
+ "-f", "1"])
+
+ error = None
try:
# Success
output = subprocess.check_output(command)
- if os.path.exists(frame_filepath):
- shutil.copy(frame_filepath, output_filepath)
- os.remove(frame_filepath)
if VERBOSE:
print(" ".join(command))
print(output.decode("utf-8"))
- return None
except subprocess.CalledProcessError as e:
# Error
- if os.path.exists(frame_filepath):
- os.remove(frame_filepath)
if VERBOSE:
print(" ".join(command))
print(e.output.decode("utf-8"))
- if b"Error: engine not found" in e.output:
- return "NO_ENGINE"
- elif b"blender probably wont start" in e.output:
- return "NO_START"
- return "CRASH"
+ error = "CRASH"
except BaseException as e:
# Crash
- if os.path.exists(frame_filepath):
- os.remove(frame_filepath)
if VERBOSE:
print(" ".join(command))
- print(e)
- return "CRASH"
+ print(e.decode("utf-8"))
+ error = "CRASH"
+
+ # Detect missing filepaths and consider those errors
+ errors = []
+ for output_filepath in output_filepaths:
+ frame_filepath = output_filepath + '0001.png'
+ if os.path.exists(frame_filepath):
+ shutil.copy(frame_filepath, output_filepath)
+ os.remove(frame_filepath)
+ errors.append(None)
+ else:
+ errors.append(error)
+ error = 'SKIPPED'
+
+ return errors
def create_argparse():
@@ -129,7 +133,7 @@ def main():
report.set_pixelated(True)
report.set_reference_dir("eevee_renders")
report.set_compare_engines('eevee', 'cycles')
- ok = report.run(test_dir, render_file)
+ ok = report.run(test_dir, render_files)
sys.exit(not ok)
diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py
index 5a2baa354c7..7dfc74904a7 100755
--- a/tests/python/modules/render_report.py
+++ b/tests/python/modules/render_report.py
@@ -374,41 +374,48 @@ class Report:
return not failed
- def _run_test(self, filepath, render_cb):
- testname = test_get_name(filepath)
- print_message(testname, 'SUCCESS', 'RUN')
- time_start = time.time()
- tmp_filepath = os.path.join(self.output_dir, "tmp_" + testname)
+ def _run_tests(self, filepaths, render_cb):
+ # Run all tests together for performance, since Blender
+ # startup time is a significant factor.
+ tmp_filepaths = []
+ for filepath in filepaths:
+ testname = test_get_name(filepath)
+ print_message(testname, 'SUCCESS', 'RUN')
+ tmp_filepaths.append(os.path.join(self.output_dir, "tmp_" + testname))
- error = render_cb(filepath, tmp_filepath)
- status = "FAIL"
- if not error:
- if not self._diff_output(filepath, tmp_filepath):
- error = "VERIFY"
+ run_errors = render_cb(filepaths, tmp_filepaths)
+ errors = []
- if os.path.exists(tmp_filepath):
- os.remove(tmp_filepath)
+ for error, filepath, tmp_filepath in zip(run_errors, filepaths, tmp_filepaths):
+ if not error:
+ if os.path.getsize(tmp_filepath) == 0:
+ error = "VERIFY"
+ elif not self._diff_output(filepath, tmp_filepath):
+ error = "VERIFY"
- time_end = time.time()
- elapsed_ms = int((time_end - time_start) * 1000)
- if not error:
- print_message("{} ({} ms)" . format(testname, elapsed_ms),
- 'SUCCESS', 'OK')
- else:
- if error == "NO_ENGINE":
- print_message("Can't perform tests because the render engine failed to load!")
- return error
- elif error == "NO_START":
- print_message('Can not perform tests because blender fails to start.',
- 'Make sure INSTALL target was r
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list