[Bf-blender-cvs] [93901e7f0a0] master: Tests: speed up render tests by running multiple in the same process

Brecht Van Lommel noreply at git.blender.org
Sat May 11 00:13:09 CEST 2019


Commit: 93901e7f0a05ba471f3b4c0201500d9dfcd68c2c
Author: Brecht Van Lommel
Date:   Fri May 10 23:00:35 2019 +0200
Branches: master
https://developer.blender.org/rB93901e7f0a05ba471f3b4c0201500d9dfcd68c2c

Tests: speed up render tests by running multiple in the same process

Blender startup time and shader compilation is a big factor when running
hundreds of tests, so now all renders in the same ctest run in the same
process. If a test crashes, the remaining tests in the same category will
be marked as skipped.

Benchmarked on a quad core with ctest -j8.

cycles: 118.1s -> 94.3s
eevee: 66.2s -> 29.2s
workbench: 31.7s -> 8.6s

===================================================================

M	tests/python/cycles_render_tests.py
M	tests/python/eevee_render_tests.py
M	tests/python/modules/render_report.py
M	tests/python/opengl_draw_tests.py
M	tests/python/workbench_render_tests.py

===================================================================

diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py
index 36f5459c2f7..a66409d85c3 100755
--- a/tests/python/cycles_render_tests.py
+++ b/tests/python/cycles_render_tests.py
@@ -9,77 +9,76 @@ import subprocess
 import sys
 
 
-def render_file(filepath, output_filepath):
-    dirname = os.path.dirname(filepath)
-    basedir = os.path.dirname(dirname)
-    subject = os.path.basename(dirname)
-
-    frame_filepath = output_filepath + '0001.png'
-
-    common_args = [
-        "-noaudio",
-        "--factory-startup",
-        "--enable-autoexec",
-        filepath,
-        "-E", "CYCLES",
-        "-o", output_filepath,
-        "-F", "PNG"]
+def render_files(filepaths, output_filepaths):
+    command = [BLENDER, "--background"]
 
     # OSL and GPU examples
     # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"]
     # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"]
     custom_args = os.getenv('CYCLESTEST_ARGS')
     custom_args = shlex.split(custom_args) if custom_args else []
-    common_args += custom_args
-
-    if subject == 'opengl':
-        command = [BLENDER, "--window-geometry", "0", "0", "1", "1"]
-        command += common_args
-        command += ['--python', os.path.join(basedir, "util", "render_opengl.py")]
-    elif subject == 'bake':
-        command = [BLENDER, "--background"]
-        command += common_args
-        command += ['--python', os.path.join(basedir, "util", "render_bake.py")]
-    elif subject == 'denoise_animation':
-        command = [BLENDER, "--background"]
-        command += common_args
-        command += ['--python', os.path.join(basedir, "util", "render_denoise.py")]
-    else:
-        command = [BLENDER, "--background"]
-        command += common_args
-        command += ["-f", "1"]
 
+    for filepath, output_filepath in zip(filepaths, output_filepaths):
+        dirname = os.path.dirname(filepath)
+        basedir = os.path.dirname(dirname)
+        subject = os.path.basename(dirname)
+
+        frame_filepath = output_filepath + '0001.png'
+
+        common_args = [
+            "-noaudio",
+            "--factory-startup",
+            "--enable-autoexec",
+            filepath,
+            "-E", "CYCLES",
+            "-o", output_filepath,
+            "-F", "PNG"]
+
+        common_args += custom_args
+
+        if subject == 'bake':
+            command.extend(common_args)
+            command.extend(['--python', os.path.join(basedir, "util", "render_bake.py")])
+        elif subject == 'denoise_animation':
+            command.extend(common_args)
+            command.extend(['--python', os.path.join(basedir, "util", "render_denoise.py")])
+        else:
+            command.extend(common_args)
+            command.extend(["-f", "1"])
+
+    error = None
     try:
         # Success
         output = subprocess.check_output(command)
-        if os.path.exists(frame_filepath):
-            shutil.copy(frame_filepath, output_filepath)
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(output.decode("utf-8"))
-        return None
     except subprocess.CalledProcessError as e:
         # Error
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(e.output.decode("utf-8"))
-        if b"Error: engine not found" in e.output:
-            return "NO_ENGINE"
-        elif b"blender probably wont start" in e.output:
-            return "NO_START"
-        return "CRASH"
+        error = "CRASH"
     except BaseException as e:
         # Crash
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
-            print(e)
-        return "CRASH"
+            print(e.decode("utf-8"))
+        error = "CRASH"
+
+    # Detect missing filepaths and consider those errors
+    errors = []
+    for output_filepath in output_filepaths:
+        frame_filepath = output_filepath + '0001.png'
+        if os.path.exists(frame_filepath):
+            shutil.copy(frame_filepath, output_filepath)
+            os.remove(frame_filepath)
+            errors.append(None)
+        else:
+            errors.append(error)
+            error = 'SKIPPED'
 
+    return errors
 
 def create_argparse():
     parser = argparse.ArgumentParser()
@@ -108,7 +107,7 @@ def main():
     report.set_pixelated(True)
     report.set_reference_dir("cycles_renders")
     report.set_compare_engines('cycles', 'eevee')
-    ok = report.run(test_dir, render_file)
+    ok = report.run(test_dir, render_files)
 
     sys.exit(not ok)
 
diff --git a/tests/python/eevee_render_tests.py b/tests/python/eevee_render_tests.py
index c0536e05164..d2c37dbcb2e 100755
--- a/tests/python/eevee_render_tests.py
+++ b/tests/python/eevee_render_tests.py
@@ -49,57 +49,61 @@ if inside_blender:
         sys.exit(1)
 
 
-def render_file(filepath, output_filepath):
-    dirname = os.path.dirname(filepath)
-    basedir = os.path.dirname(dirname)
-    subject = os.path.basename(dirname)
-
-    frame_filepath = output_filepath + '0001.png'
-
+def render_files(filepaths, output_filepaths):
     command = [
         BLENDER,
         "--background",
         "-noaudio",
         "--factory-startup",
-        "--enable-autoexec",
-        filepath,
-        "-E", "BLENDER_EEVEE",
-        "-P",
-        os.path.realpath(__file__),
-        "-o", output_filepath,
-        "-F", "PNG",
-        "-f", "1"]
+        "--enable-autoexec"]
+
+    for filepath, output_filepath in zip(filepaths, output_filepaths):
+        frame_filepath = output_filepath + '0001.png'
+        if os.path.exists(frame_filepath):
+            os.remove(frame_filepath)
 
+        command.extend([
+            filepath,
+            "-E", "BLENDER_EEVEE",
+            "-P",
+            os.path.realpath(__file__),
+            "-o", output_filepath,
+            "-F", "PNG",
+            "-f", "1"])
+
+    error = None
     try:
         # Success
         output = subprocess.check_output(command)
-        if os.path.exists(frame_filepath):
-            shutil.copy(frame_filepath, output_filepath)
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(output.decode("utf-8"))
-        return None
     except subprocess.CalledProcessError as e:
         # Error
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(e.output.decode("utf-8"))
-        if b"Error: engine not found" in e.output:
-            return "NO_ENGINE"
-        elif b"blender probably wont start" in e.output:
-            return "NO_START"
-        return "CRASH"
+        error = "CRASH"
     except BaseException as e:
         # Crash
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
-            print(e)
-        return "CRASH"
+            print(e.decode("utf-8"))
+        error = "CRASH"
+
+    # Detect missing filepaths and consider those errors
+    errors = []
+    for output_filepath in output_filepaths:
+        frame_filepath = output_filepath + '0001.png'
+        if os.path.exists(frame_filepath):
+            shutil.copy(frame_filepath, output_filepath)
+            os.remove(frame_filepath)
+            errors.append(None)
+        else:
+            errors.append(error)
+            error = 'SKIPPED'
+
+    return errors
 
 
 def create_argparse():
@@ -129,7 +133,7 @@ def main():
     report.set_pixelated(True)
     report.set_reference_dir("eevee_renders")
     report.set_compare_engines('eevee', 'cycles')
-    ok = report.run(test_dir, render_file)
+    ok = report.run(test_dir, render_files)
 
     sys.exit(not ok)
 
diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py
index 5a2baa354c7..7dfc74904a7 100755
--- a/tests/python/modules/render_report.py
+++ b/tests/python/modules/render_report.py
@@ -374,41 +374,48 @@ class Report:
 
         return not failed
 
-    def _run_test(self, filepath, render_cb):
-        testname = test_get_name(filepath)
-        print_message(testname, 'SUCCESS', 'RUN')
-        time_start = time.time()
-        tmp_filepath = os.path.join(self.output_dir, "tmp_" + testname)
+    def _run_tests(self, filepaths, render_cb):
+        # Run all tests together for performance, since Blender
+        # startup time is a significant factor.
+        tmp_filepaths = []
+        for filepath in filepaths:
+            testname = test_get_name(filepath)
+            print_message(testname, 'SUCCESS', 'RUN')
+            tmp_filepaths.append(os.path.join(self.output_dir, "tmp_" + testname))
 
-        error = render_cb(filepath, tmp_filepath)
-        status = "FAIL"
-        if not error:
-            if not self._diff_output(filepath, tmp_filepath):
-                error = "VERIFY"
+        run_errors = render_cb(filepaths, tmp_filepaths)
+        errors = []
 
-        if os.path.exists(tmp_filepath):
-            os.remove(tmp_filepath)
+        for error, filepath, tmp_filepath in zip(run_errors, filepaths, tmp_filepaths):
+            if not error:
+                if os.path.getsize(tmp_filepath) == 0:
+                    error = "VERIFY"
+                elif not self._diff_output(filepath, tmp_filepath):
+                    error = "VERIFY"
 
-        time_end = time.time()
-        elapsed_ms = int((time_end - time_start) * 1000)
-        if not error:
-            print_message("{} ({} ms)" . format(testname, elapsed_ms),
-                          'SUCCESS', 'OK')
-        else:
-            if error == "NO_ENGINE":
-                print_message("Can't perform tests because the render engine failed to load!")
-                return error
-            elif error == "NO_START":
-                print_message('Can not perform tests because blender fails to start.',
-                              'Make sure INSTALL target was r

@@ Diff output truncated at 10240 characters. @@



More information about the Bf-blender-cvs mailing list