[Bf-blender-cvs] [6f516fcc631] master: Tests: speed up render tests by running multiple in the same process

Brecht Van Lommel noreply at git.blender.org
Thu Jun 27 12:46:19 CEST 2019


Commit: 6f516fcc631c177a1c9f62b3a22e0c4e0e9713d4
Author: Brecht Van Lommel
Date:   Fri May 10 23:00:35 2019 +0200
Branches: master
https://developer.blender.org/rB6f516fcc631c177a1c9f62b3a22e0c4e0e9713d4

Tests: speed up render tests by running multiple in the same process

Blender startup time and shader compilation is a big factor when running
hundreds of tests, so now all renders in the same ctest run in the same
process.

This was previously reverted due to skipping other tests when one test
crashed. Now if a test crashes, Blender is re-run with the remaining
tests so we get results from them still.

===================================================================

M	tests/python/cycles_render_tests.py
M	tests/python/eevee_render_tests.py
M	tests/python/modules/render_report.py
M	tests/python/opengl_draw_tests.py
M	tests/python/workbench_render_tests.py

===================================================================

diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py
index 349ba83f88a..79ba11fdd44 100755
--- a/tests/python/cycles_render_tests.py
+++ b/tests/python/cycles_render_tests.py
@@ -9,14 +9,13 @@ import subprocess
 import sys
 
 
-def render_file(filepath, output_filepath):
+def get_arguments(filepath, output_filepath):
     dirname = os.path.dirname(filepath)
     basedir = os.path.dirname(dirname)
     subject = os.path.basename(dirname)
 
-    frame_filepath = output_filepath + '0001.png'
-
-    common_args = [
+    args = [
+        "--background",
         "-noaudio",
         "--factory-startup",
         "--enable-autoexec",
@@ -29,57 +28,17 @@ def render_file(filepath, output_filepath):
     # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"]
     # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"]
     custom_args = os.getenv('CYCLESTEST_ARGS')
-    custom_args = shlex.split(custom_args) if custom_args else []
-    common_args += custom_args
+    if custom_args:
+        args.extend(shlex.split(custom_args))
 
-    if subject == 'opengl':
-        command = [BLENDER, "--window-geometry", "0", "0", "1", "1"]
-        command += common_args
-        command += ['--python', os.path.join(basedir, "util", "render_opengl.py")]
-    elif subject == 'bake':
-        command = [BLENDER, "--background"]
-        command += common_args
-        command += ['--python', os.path.join(basedir, "util", "render_bake.py")]
+    if subject == 'bake':
+        args.extend(['--python', os.path.join(basedir, "util", "render_bake.py")])
     elif subject == 'denoise_animation':
-        command = [BLENDER, "--background"]
-        command += common_args
-        command += ['--python', os.path.join(basedir, "util", "render_denoise.py")]
+        args.extend(['--python', os.path.join(basedir, "util", "render_denoise.py")])
     else:
-        command = [BLENDER, "--background"]
-        command += common_args
-        command += ["-f", "1"]
-
-    try:
-        # Success
-        output = subprocess.check_output(command)
-        if os.path.exists(frame_filepath):
-            shutil.copy(frame_filepath, output_filepath)
-            os.remove(frame_filepath)
-        if VERBOSE:
-            print(" ".join(command))
-            print(output.decode("utf-8"))
-        return None
-    except subprocess.CalledProcessError as e:
-        # Error
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
-        if VERBOSE:
-            print(" ".join(command))
-            print(e.output.decode("utf-8"))
-        if b"Error: engine not found" in e.output:
-            return "NO_ENGINE"
-        elif b"blender probably wont start" in e.output:
-            return "NO_START"
-        return "CRASH"
-    except BaseException as e:
-        # Crash
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
-        if VERBOSE:
-            print(" ".join(command))
-            print(e)
-        return "CRASH"
+        args.extend(["-f", "1"])
 
+    return args
 
 def create_argparse():
     parser = argparse.ArgumentParser()
@@ -94,11 +53,7 @@ def main():
     parser = create_argparse()
     args = parser.parse_args()
 
-    global BLENDER, VERBOSE
-
-    BLENDER = args.blender[0]
-    VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
-
+    blender = args.blender[0]
     test_dir = args.testdir[0]
     idiff = args.idiff[0]
     output_dir = args.outdir[0]
@@ -108,7 +63,7 @@ def main():
     report.set_pixelated(True)
     report.set_reference_dir("cycles_renders")
     report.set_compare_engines('cycles', 'eevee')
-    ok = report.run(test_dir, render_file)
+    ok = report.run(test_dir, blender, get_arguments, batch=True)
 
     sys.exit(not ok)
 
diff --git a/tests/python/eevee_render_tests.py b/tests/python/eevee_render_tests.py
index 9fb8a5f41a8..0875e4b171a 100755
--- a/tests/python/eevee_render_tests.py
+++ b/tests/python/eevee_render_tests.py
@@ -8,7 +8,6 @@ import shutil
 import subprocess
 import sys
 
-
 def setup():
     import bpy
 
@@ -46,15 +45,8 @@ if inside_blender:
         sys.exit(1)
 
 
-def render_file(filepath, output_filepath):
-    dirname = os.path.dirname(filepath)
-    basedir = os.path.dirname(dirname)
-    subject = os.path.basename(dirname)
-
-    frame_filepath = output_filepath + '0001.png'
-
-    command = [
-        BLENDER,
+def get_arguments(filepath, output_filepath):
+    return [
         "--background",
         "-noaudio",
         "--factory-startup",
@@ -67,37 +59,6 @@ def render_file(filepath, output_filepath):
         "-F", "PNG",
         "-f", "1"]
 
-    try:
-        # Success
-        output = subprocess.check_output(command)
-        if os.path.exists(frame_filepath):
-            shutil.copy(frame_filepath, output_filepath)
-            os.remove(frame_filepath)
-        if VERBOSE:
-            print(" ".join(command))
-            print(output.decode("utf-8"))
-        return None
-    except subprocess.CalledProcessError as e:
-        # Error
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
-        if VERBOSE:
-            print(" ".join(command))
-            print(e.output.decode("utf-8"))
-        if b"Error: engine not found" in e.output:
-            return "NO_ENGINE"
-        elif b"blender probably wont start" in e.output:
-            return "NO_START"
-        return "CRASH"
-    except BaseException as e:
-        # Crash
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
-        if VERBOSE:
-            print(" ".join(command))
-            print(e)
-        return "CRASH"
-
 
 def create_argparse():
     parser = argparse.ArgumentParser()
@@ -112,11 +73,7 @@ def main():
     parser = create_argparse()
     args = parser.parse_args()
 
-    global BLENDER, VERBOSE
-
-    BLENDER = args.blender[0]
-    VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
-
+    blender = args.blender[0]
     test_dir = args.testdir[0]
     idiff = args.idiff[0]
     output_dir = args.outdir[0]
@@ -126,7 +83,7 @@ def main():
     report.set_pixelated(True)
     report.set_reference_dir("eevee_renders")
     report.set_compare_engines('eevee', 'cycles')
-    ok = report.run(test_dir, render_file)
+    ok = report.run(test_dir, blender, get_arguments, batch=True)
 
     sys.exit(not ok)
 
diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py
index 2f99a3b6292..15826f97400 100755
--- a/tests/python/modules/render_report.py
+++ b/tests/python/modules/render_report.py
@@ -134,10 +134,10 @@ class Report:
     def set_compare_engines(self, engine, other_engine):
         self.compare_engines = (engine, other_engine)
 
-    def run(self, dirpath, render_cb):
+    def run(self, dirpath, blender, arguments_cb, batch=False):
         # Run tests and output report.
         dirname = os.path.basename(dirpath)
-        ok = self._run_all_tests(dirname, dirpath, render_cb)
+        ok = self._run_all_tests(dirname, dirpath, blender, arguments_cb, batch)
         self._write_data(dirname)
         self._write_html()
         if self.compare_engines:
@@ -399,43 +399,81 @@ class Report:
 
         return not failed
 
-    def _run_test(self, filepath, render_cb):
-        testname = test_get_name(filepath)
-        print_message(testname, 'SUCCESS', 'RUN')
-        time_start = time.time()
-        tmp_filepath = os.path.join(self.output_dir, "tmp_" + testname)
+    def _run_tests(self, filepaths, blender, arguments_cb, batch):
+        # Run multiple tests in a single Blender process since startup can be
+        # a significant factor. In case of crashes, re-run the remaining tests.
+        verbose = os.environ.get("BLENDER_VERBOSE") is not None
 
-        error = render_cb(filepath, tmp_filepath)
-        status = "FAIL"
-        if not error:
-            if not self._diff_output(filepath, tmp_filepath):
-                error = "VERIFY"
+        remaining_filepaths = filepaths[:]
+        errors = []
 
-        if os.path.exists(tmp_filepath):
-            os.remove(tmp_filepath)
+        while len(remaining_filepaths) > 0:
+            command = [blender]
+            output_filepaths = []
 
-        time_end = time.time()
-        elapsed_ms = int((time_end - time_start) * 1000)
-        if not error:
-            print_message("{} ({} ms)" . format(testname, elapsed_ms),
-                          'SUCCESS', 'OK')
-        else:
-            if error == "NO_ENGINE":
-                print_message("Can't perform tests because the render engine failed to load!")
-                return error
-            elif error == "NO_START":
-                print_message('Can not perform tests because blender fails to start.',
-                              'Make sure INSTALL target was run.')
-                return error
-            elif error == 'VERIFY':
-                print_message("Rendered result is different from reference image")
-            else:
-                print_message("Unknown error %r" % error)
-            print_message("{} ({} ms)" . format(testname, elapsed_ms),
-                          'FAILURE', 'FAILED')
-        return error
+            # Construct output filepaths and command to run
+            for filepath in remaining_filepaths:
+                testname = test_get_name(filepath)
+                print_message(testname, 'SUCCESS', 'RUN')
 
-    def _run_all_tests(self, dirname, dirpath, render_cb):
+                base_output_filepath = os.path.join(self.output_dir, "tmp_" + testname)
+                output_filepath = base_output_filepath + '0001.png'
+                output_filepaths.append(output_filepath)
+
+                if os.path.exists(output_filepath):
+                    os.remove(output_filepath)
+
+                command.extend(arguments_cb(filepath, base_output_filepath))
+
+                # Only chain multiple commands for batch
+                if not batch:
+                    break
+
+            # Run process
+            crash = False
+            try:
+                output = subprocess.check_out

@@ Diff output truncated at 10240 characters. @@



More information about the Bf-blender-cvs mailing list