[Bf-blender-cvs] [cae72ca] master: CTests: Add render tests for Cycles

[Sergey Sharybin]

Commit: cae72caa82714107909fbfeb1ebc09653dce0767
Author: Sergey Sharybin
Date:   Thu Jan 22 15:53:49 2015 +0500
Branches: master
https://developer.blender.org/rBcae72caa82714107909fbfeb1ebc09653dce0767

CTests: Add render tests for Cycles

The idea is to use the set of really small images from the lib folder
and run Cycles render on them comparing render output to reference
images in the tests repository.

For sure same thing could become more generic for BI or Freestyle
render engines.

Thanks Campbell for review and code tweaks!

===================================================================

M	tests/python/CMakeLists.txt
A	tests/python/cycles_render_tests.py

===================================================================

diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt
index 85c6869..59d3aa1 100644
--- a/tests/python/CMakeLists.txt
+++ b/tests/python/CMakeLists.txt
@@ -48,6 +48,7 @@ else()
 endif()
 
 # for testing with valgrind prefix: valgrind --track-origins=yes --error-limit=no 
+set(TEST_BLENDER_EXE_BARE ${TEST_BLENDER_EXE})
 set(TEST_BLENDER_EXE ${TEST_BLENDER_EXE} --background -noaudio --factory-startup --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts)
 
 
@@ -358,3 +359,14 @@ add_test(export_fbx_all_objects ${TEST_BLENDER_EXE}
 	--md5_source=${TEST_OUT_DIR}/export_fbx_all_objects.fbx
 	--md5=b35eb2a9d0e73762ecae2278c25a38ac --md5_method=FILE
 )
+
+if(WITH_CYCLES)
+	if(OPENIMAGEIO_IDIFF)
+		add_test(cycles_shaders_test
+			${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py
+			-blender "${TEST_BLENDER_EXE_BARE}"
+			-testdir "${TEST_SRC_DIR}/cycles/ctests/shader"
+			-idiff "${OPENIMAGEIO_IDIFF}"
+		)
+	endif()
+endif()
diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py
new file mode 100755
index 0000000..815811f
--- /dev/null
+++ b/tests/python/cycles_render_tests.py
@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+# Apache License, Version 2.0
+
+import argparse
+import os
+import subprocess
+import sys
+import tempfile
+
+
+def render_file(filepath):
+    command = (
+        BLENDER,
+        "--background",
+        "-noaudio",
+        "--factory-startup",
+        filepath,
+        "-E", "CYCLES",
+        "-o", TEMP_FILE_MASK,
+        "-F", "PNG",
+        "-f", "1",
+        )
+    try:
+        output = subprocess.check_output(command)
+        if VERBOSE:
+            print(output.decode("utf-8"))
+        return None
+    except subprocess.CalledProcessError as e:
+        if os.path.exists(TEMP_FILE):
+            os.remove(TEMP_FILE)
+        if VERBOSE:
+            print(e.output.decode("utf-8"))
+        if b"Error: engine not found" in e.output:
+            return "NO_CYCLES"
+        elif b"blender probably wont start" in e.output:
+            return "NO_START"
+        return "CRASH"
+    except:
+        if os.path.exists(TEMP_FILE):
+            os.remove(TEMP_FILE)
+        if VERBOSE:
+            print(e.output.decode("utf-8"))
+        return "CRASH"
+
+
+def test_get_name(filepath):
+    filename = os.path.basename(filepath)
+    return os.path.splitext(filename)[0]
+
+
+def verify_output(filepath):
+    testname = test_get_name(filepath)
+    dirpath = os.path.dirname(filepath)
+    reference_dirpath = os.path.join(dirpath, "reference_renders")
+    reference_image = os.path.join(reference_dirpath, testname + ".png")
+    if not os.path.exists(reference_image):
+        return False
+    command = (
+        IDIFF,
+        "-fail", "0.01",
+        "-failpercent", "1",
+        reference_image,
+        TEMP_FILE,
+        )
+    try:
+        subprocess.check_output(command)
+        return True
+    except subprocess.CalledProcessError as grepexc:
+        return grepexc.returncode == 1
+
+
+def run_test(filepath):
+    testname = test_get_name(filepath)
+    spacer = "." * (32 - len(testname))
+    print(testname, spacer, end="")
+    sys.stdout.flush()
+    error = render_file(filepath)
+    if not error:
+        if verify_output(filepath):
+            print("PASS")
+        else:
+            error = "VERIFY"
+    if error:
+        print("FAIL", error)
+    return error
+
+
+def blend_list(path):
+    for dirpath, dirnames, filenames in os.walk(path):
+        for filename in filenames:
+            if filename.lower().endswith(".blend"):
+                filepath = os.path.join(dirpath, filename)
+                yield filepath
+
+
+def run_all_tests(dirpath):
+    failed_tests = []
+    all_files = list(blend_list(dirpath))
+    all_files.sort()
+    for filepath in all_files:
+        error = run_test(filepath)
+        if error:
+            if error == "NO_CYCLES":
+                print("Can't perform tests because Cycles failed to load!")
+                return False
+            elif error == "NO_START":
+                print('Can not perform tests because blender fails to start.',
+                      'Make sure INSTALL target was run.')
+                return False
+            else:
+                print("Unknown error %r" % error)
+            testname = test_get_name(filepath)
+            failed_tests.append(testname)
+    if failed_tests:
+        failed_tests.sort()
+        print("\n\nFAILED tests:")
+        for test in failed_tests:
+            print("   ", test)
+        return False
+    return True
+
+
+def create_argparse():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-blender", nargs="+")
+    parser.add_argument("-testdir", nargs=1)
+    parser.add_argument("-idiff", nargs=1)
+    return parser
+
+
+def main():
+    parser = create_argparse()
+    args = parser.parse_args()
+
+    global BLENDER, ROOT, IDIFF
+    global TEMP_FILE, TEMP_FILE_MASK, TEST_SCRIPT
+    global VERBOSE
+
+    BLENDER = args.blender[0]
+    ROOT = args.testdir[0]
+    IDIFF = args.idiff[0]
+
+    TEMP = tempfile.mkdtemp()
+    TEMP_FILE_MASK = os.path.join(TEMP, "test")
+    TEMP_FILE = TEMP_FILE_MASK + "0001.png"
+
+    TEST_SCRIPT = os.path.join(os.path.dirname(__file__), "runtime_check.py")
+
+    VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
+
+    ok = run_all_tests(ROOT)
+
+    # Cleanup temp files and folders
+    if os.path.exists(TEMP_FILE):
+        os.remove(TEMP_FILE)
+    os.rmdir(TEMP)
+
+    sys.exit(not ok)
+
+
+if __name__ == "__main__":
+    main()