[Bf-blender-cvs] [a31a0380be6] cycles-x: Tests: run render tests for fixed 10s, and measure time per sample

[Brecht Van Lommel]

Commit: a31a0380be6dff2c5bdde3aed8fecdcc0d2f5304
Author: Brecht Van Lommel
Date:   Wed Jul 21 16:56:22 2021 +0200
Branches: cycles-x
https://developer.blender.org/rBa31a0380be6dff2c5bdde3aed8fecdcc0d2f5304

Tests: run render tests for fixed 10s, and measure time per sample

Taking advantage of the new time limit rendering to make the benchmark runtime
the same on all devices.

For scenes with adaptive sampling it still uses the samples specified in the
file, since that makes time per sample an unreliable metric.

===================================================================

M	tests/performance/tests/animation.py
M	tests/performance/tests/cycles.py

===================================================================

diff --git a/tests/performance/tests/animation.py b/tests/performance/tests/animation.py
index 1a92f1a9718..614876bbbd3 100644
--- a/tests/performance/tests/animation.py
+++ b/tests/performance/tests/animation.py
@@ -9,14 +9,20 @@ def _run(args):
     import time
 
     start_time = time.time()
+    elapsed_time = 0.0
+    num_frames = 0
 
-    scene = bpy.context.scene
-    for i in range(scene.frame_start, scene.frame_end):
-        scene.frame_set(scene.frame_start)
+    while elapsed_time < 10.0:
+        scene = bpy.context.scene
+        for i in range(scene.frame_start, scene.frame_end + 1):
+            scene.frame_set(scene.frame_start)
 
-    elapsed_time = time.time() - start_time
+        num_frames += scene.frame_end + 1 - scene.frame_start
+        elapsed_time = time.time() - start_time
 
-    result = {'time': elapsed_time}
+    time_per_frame = elapsed_time / num_frames
+
+    result = {'time': time_per_frame}
     return result
 
 
diff --git a/tests/performance/tests/cycles.py b/tests/performance/tests/cycles.py
index bac6b8a7ceb..78e3c4bb19d 100644
--- a/tests/performance/tests/cycles.py
+++ b/tests/performance/tests/cycles.py
@@ -17,6 +17,16 @@ def _run(args):
     scene.render.image_settings.file_format = 'PNG'
     scene.cycles.device = 'CPU' if device_type == 'CPU' else 'GPU'
 
+    if scene.cycles.use_adaptive_sampling:
+        # Render samples specified in file, no other way to measure
+        # adaptive sampling performance reliably.
+        scene.cycles.time_limit = 0.0
+    else:
+        # Render for fixed amount of time so it's adaptive to the
+        # machine and devices.
+        scene.cycles.samples = 16384
+        scene.cycles.time_limit = 10.0
+
     if scene.cycles.device == 'GPU':
         # Enable specified GPU in preferences.
         prefs = bpy.context.preferences
@@ -62,12 +72,14 @@ class CyclesTest(api.Test):
                 'device_index': device_index,
                 'render_filepath': str(env.log_file.parent / (env.log_file.stem + '.png'))}
 
-        _, lines = env.run_in_blender(_run, args, ['--debug-cycles', '--verbose', '1', self.filepath])
+        _, lines = env.run_in_blender(_run, args, ['--debug-cycles', '--verbose', '2', self.filepath])
 
         # Parse render time from output
         prefix_time = "Render time (without synchronization): "
         prefix_memory = "Peak: "
+        prefix_time_per_sample = "Average time per sample: "
         time = None
+        time_per_sample = None
         memory = None
         for line in lines:
             line = line.strip()
@@ -75,12 +87,20 @@ class CyclesTest(api.Test):
             if offset != -1:
                 time = line[offset + len(prefix_time):]
                 time = float(time)
+            offset = line.find(prefix_time_per_sample)
+            if offset != -1:
+                time_per_sample = line[offset + len(prefix_time_per_sample):]
+                time_per_sample = time_per_sample.split()[0]
+                time_per_sample = float(time_per_sample)
             offset = line.find(prefix_memory)
             if offset != -1:
                 memory = line[offset + len(prefix_memory):]
                 memory = memory.split()[0].replace(',', '')
                 memory = float(memory)
 
+        if time_per_sample:
+            time = time_per_sample
+
         if not (time and memory):
             raise Exception("Error parsing render time output")