[Bf-blender-cvs] [0f3f093d3b3] master: Cycles: add HTML report to inspect failed test images.
Brecht Van Lommel
noreply at git.blender.org
Mon Aug 7 14:03:49 CEST 2017
Commit: 0f3f093d3b3d1b2fb9593233f23665d3135bea2d
Author: Brecht Van Lommel
Date: Thu Aug 3 16:41:50 2017 +0200
Branches: master
https://developer.blender.org/rB0f3f093d3b3d1b2fb9593233f23665d3135bea2d
Cycles: add HTML report to inspect failed test images.
Shows new, reference and diff renders, with mouse hover to flip between
new and ref for easy comparison. This generates a report.html in
build_dir/tests/cycles, stored along with the new and diff images.
Differential Revision: https://developer.blender.org/D2770
===================================================================
M tests/python/CMakeLists.txt
M tests/python/cycles_render_tests.py
===================================================================
diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt
index f2fea48d9f4..4ba89c2805d 100644
--- a/tests/python/CMakeLists.txt
+++ b/tests/python/CMakeLists.txt
@@ -518,6 +518,7 @@ if(WITH_CYCLES)
-blender "$<TARGET_FILE:blender>"
-testdir "${TEST_SRC_DIR}/cycles/ctests/${subject}"
-idiff "${OPENIMAGEIO_IDIFF}"
+ -outdir "${TEST_OUT_DIR}/cycles"
)
else()
add_test(
@@ -526,6 +527,7 @@ if(WITH_CYCLES)
-blender "$<TARGET_FILE:blender>"
-testdir "${TEST_SRC_DIR}/cycles/ctests/${subject}"
-idiff "${OPENIMAGEIO_IDIFF}"
+ -outdir "${TEST_OUT_DIR}/cycles"
)
endif()
endmacro()
diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py
index a030cc5e0de..ea84f27ab7e 100755
--- a/tests/python/cycles_render_tests.py
+++ b/tests/python/cycles_render_tests.py
@@ -2,7 +2,9 @@
# Apache License, Version 2.0
import argparse
+import glob
import os
+import pathlib
import shutil
import subprocess
import sys
@@ -24,7 +26,7 @@ class COLORS_DUMMY:
COLORS = COLORS_DUMMY
-def printMessage(type, status, message):
+def print_message(message, type=None, status=''):
if type == 'SUCCESS':
print(COLORS.GREEN, end="")
elif type == 'FAILURE':
@@ -109,20 +111,126 @@ def test_get_name(filepath):
filename = os.path.basename(filepath)
return os.path.splitext(filename)[0]
-
-def verify_output(filepath):
+def test_get_images(filepath):
testname = test_get_name(filepath)
dirpath = os.path.dirname(filepath)
- reference_dirpath = os.path.join(dirpath, "reference_renders")
- reference_image = os.path.join(reference_dirpath, testname + ".png")
- failed_image = os.path.join(reference_dirpath, testname + ".fail.png")
- if not os.path.exists(reference_image):
+ ref_dirpath = os.path.join(dirpath, "reference_renders")
+ ref_img = os.path.join(ref_dirpath, testname + ".png")
+ new_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath))
+ if not os.path.exists(new_dirpath):
+ os.makedirs(new_dirpath)
+ new_img = os.path.join(new_dirpath, testname + ".png")
+ diff_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "diff")
+ if not os.path.exists(diff_dirpath):
+ os.makedirs(diff_dirpath)
+ diff_img = os.path.join(diff_dirpath, testname + ".diff.png")
+ return ref_img, new_img, diff_img
+
+
+class Report:
+ def __init__(self, testname):
+ self.failed_tests = ""
+ self.passed_tests = ""
+ self.testname = testname
+
+ def output(self):
+ # write intermediate data for single test
+ outdir = os.path.join(OUTDIR, self.testname)
+ f = open(os.path.join(outdir, "failed.data"), "w")
+ f.write(self.failed_tests)
+ f.close()
+
+ f = open(os.path.join(outdir, "passed.data"), "w")
+ f.write(self.passed_tests)
+ f.close()
+
+ # gather intermediate data for all tests
+ failed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/failed.data")))
+ passed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/passed.data")))
+
+ failed_tests = ""
+ passed_tests = ""
+
+ for filename in failed_data:
+ failed_tests += open(os.path.join(OUTDIR, filename), "r").read()
+ for filename in passed_data:
+ passed_tests += open(os.path.join(OUTDIR, filename), "r").read()
+
+ # write html for all tests
+ self.html = """
+<html>
+<head>
+ <title>Cycles Test Report</title>
+ <style>
+ img {{ image-rendering: pixelated; width: 256; background-color: #000; }}
+ table td:first-child {{ width: 100%; }}
+ </style>
+ <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css">
+</head>
+<body>
+ <div class="container">
+ <br/>
+ <h1>Cycles Test Report</h1>
+ <br/>
+ <table class="table table-striped">
+ <thead class="thead-default">
+ <tr><th>Name</th><th>New</th><th>Reference</th><th>Diff</th>
+ </thead>
+ {}{}
+ </table>
+ <br/>
+ </div>
+</body>
+</html>
+ """ . format(failed_tests, passed_tests)
+
+ filepath = os.path.join(OUTDIR, "report.html")
+ f = open(filepath, "w")
+ f.write(self.html)
+ f.close()
+
+ print_message("Report saved to: " + pathlib.Path(filepath).as_uri())
+
+ def add_test(self, filepath, error):
+ name = test_get_name(filepath)
+
+ ref_img, new_img, diff_img = test_get_images(filepath)
+
+ status = error if error else ""
+ style = """ style="background-color: #f99;" """ if error else ""
+
+ new_url = pathlib.Path(new_img).as_uri()
+ ref_url = pathlib.Path(ref_img).as_uri()
+ diff_url = pathlib.Path(diff_img).as_uri()
+
+ test_html = """
+ <tr{}>
+ <td><b>{}</b><br/>{}<br/>{}</td>
+ <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';"></td>
+ <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';"></td>
+ <td><img src="{}"></td>
+ </tr>""" . format(style, name, self.testname, status,
+ new_url, ref_url, new_url,
+ ref_url, new_url, ref_url,
+ diff_url)
+
+ if error:
+ self.failed_tests += test_html
+ else:
+ self.passed_tests += test_html
+
+
+def verify_output(report, filepath):
+ ref_img, new_img, diff_img = test_get_images(filepath)
+ if not os.path.exists(ref_img):
return False
+
+ # diff test with threshold
command = (
IDIFF,
- "-fail", "0.015",
+ "-fail", "0.016",
"-failpercent", "1",
- reference_image,
+ ref_img,
TEMP_FILE,
)
try:
@@ -130,47 +238,66 @@ def verify_output(filepath):
failed = False
except subprocess.CalledProcessError as e:
if VERBOSE:
- print(e.output.decode("utf-8"))
+ print_message(e.output.decode("utf-8"))
failed = e.returncode != 1
- if failed:
- shutil.copy(TEMP_FILE, failed_image)
- elif os.path.exists(failed_image):
- os.remove(failed_image)
+
+ # generate diff image
+ command = (
+ IDIFF,
+ "-o", diff_img,
+ "-abs", "-scale", "16",
+ ref_img,
+ TEMP_FILE
+ )
+
+ try:
+ subprocess.check_output(command)
+ except subprocess.CalledProcessError as e:
+ if VERBOSE:
+ print_message(e.output.decode("utf-8"))
+
+ # copy new image
+ if os.path.exists(new_img):
+ os.remove(new_img)
+ if os.path.exists(TEMP_FILE):
+ shutil.copy(TEMP_FILE, new_img)
+
return not failed
-def run_test(filepath):
+def run_test(report, filepath):
testname = test_get_name(filepath)
spacer = "." * (32 - len(testname))
- printMessage('SUCCESS', 'RUN', testname)
+ print_message(testname, 'SUCCESS', 'RUN')
time_start = time.time()
error = render_file(filepath)
status = "FAIL"
if not error:
- if not verify_output(filepath):
+ if not verify_output(report, filepath):
error = "VERIFY"
time_end = time.time()
elapsed_ms = int((time_end - time_start) * 1000)
if not error:
- printMessage('SUCCESS', 'OK', "{} ({} ms)" .
- format(testname, elapsed_ms))
+ print_message("{} ({} ms)" . format(testname, elapsed_ms),
+ 'SUCCESS', 'OK')
else:
if error == "NO_CYCLES":
- print("Can't perform tests because Cycles failed to load!")
- return False
+ print_message("Can't perform tests because Cycles failed to load!")
+ return error
elif error == "NO_START":
- print('Can not perform tests because blender fails to start.',
+ print_message('Can not perform tests because blender fails to start.',
'Make sure INSTALL target was run.')
- return False
+ return error
elif error == 'VERIFY':
- print("Rendered result is different from reference image")
+ print_message("Rendered result is different from reference image")
else:
- print("Unknown error %r" % error)
- printMessage('FAILURE', 'FAILED', "{} ({} ms)" .
- format(testname, elapsed_ms))
+ print_message("Unknown error %r" % error)
+ print_message("{} ({} ms)" . format(testname, elapsed_ms),
+ 'FAILURE', 'FAILED')
return error
+
def blend_list(path):
for dirpath, dirnames, filenames in os.walk(path):
for filename in filenames:
@@ -178,17 +305,18 @@ def blend_list(path):
filepath = os.path.join(dirpath, filename)
yield filepath
-
def run_all_tests(dirpath):
passed_tests = []
failed_tests = []
all_files = list(blend_list(dirpath))
all_files.sort()
- printMessage('SUCCESS', "==========",
- "Running {} tests from 1 test case." . format(len(all_files)))
+ report = Report(os.path.basename(dirpath))
+ print_message("Running {} tests from 1 test case." .
+ format(len(all_files)),
+ 'SUCCESS', "==========")
time_start = time.time()
for filepath in all_files:
- error = run_test(filepath)
+ error = run_test(report, filepath)
testname = test_get_name(filepath)
if error:
if error == "NO_CYCLES":
@@ -198,28 +326,33 @@ def run_all_tests(dirpath):
failed_tests.append(testname)
else:
passed_tests.append(testname)
+ report.add_test(filepath, error)
time_end = time.time()
elapsed_ms = int((time_end - time_start) * 1000)
- print("")
- printMessage('SUCCESS', "==========",
- "{} tests from 1 test case ran. ({} ms total)" .
- format(len(all_files), elapsed_ms))
- printMessage('SUCCESS', 'PASSED', "{} tes
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list