[Bf-blender-cvs] SVN commit: /data/svn/bf-blender [40444] trunk/blender/build_files/cmake: use the systems number of processes for running static checks - multiple jobs .

[Campbell Barton]

Revision: 40444
          http://projects.blender.org/scm/viewvc.php?view=rev&root=bf-blender&revision=40444
Author:   campbellbarton
Date:     2011-09-22 04:41:12 +0000 (Thu, 22 Sep 2011)
Log Message:
-----------
use the systems number of processes for running static checks - multiple jobs.

Modified Paths:
--------------
    trunk/blender/build_files/cmake/cmake_static_check_cppcheck.py
    trunk/blender/build_files/cmake/cmake_static_check_sparse.py
    trunk/blender/build_files/cmake/cmake_static_check_splint.py
    trunk/blender/build_files/cmake/project_source_info.py

Modified: trunk/blender/build_files/cmake/cmake_static_check_cppcheck.py
===================================================================
--- trunk/blender/build_files/cmake/cmake_static_check_cppcheck.py	2011-09-22 03:14:50 UTC (rev 40443)
+++ trunk/blender/build_files/cmake/cmake_static_check_cppcheck.py	2011-09-22 04:41:12 UTC (rev 40444)
@@ -37,7 +37,7 @@
 
 CHECKER_ARGS = [
     # not sure why this is needed, but it is.
-    "-I" + os.path.join(project_source_info.SOURCE_DIR, "extern/glew/include"),
+    "-I" + os.path.join(project_source_info.SOURCE_DIR, "extern", "glew", "include"),
 
     #  "--check-config", # when includes are missing
     #  "--enable=all",  # if you want sixty hundred pedantic suggestions
@@ -58,20 +58,22 @@
 
         check_commands.append((c, cmd))
 
-    for i, (c, cmd) in enumerate(check_commands):
+
+    process_functions = []
+    def my_process(i, c, cmd):
         percent = 100.0 * (i / (len(check_commands) - 1))
         percent_str = "[" + ("%.2f]" % percent).rjust(7) + " %:"
 
-        # if percent < 27.9:
-        #    continue
-
-        # let cppcheck finish the line off...
+        sys.stdout.flush()
         sys.stdout.write("%s " % percent_str)
 
-        sys.stdout.flush()
-        process = subprocess.Popen(cmd)
-        process.wait()
+        return subprocess.Popen(cmd)
 
+    for i, (c, cmd) in enumerate(check_commands):
+        process_functions.append((my_process, (i, c, cmd)))
 
+    project_source_info.queue_processes(process_functions)
+
+
 if __name__ == "__main__":
     main()

Modified: trunk/blender/build_files/cmake/cmake_static_check_sparse.py
===================================================================
--- trunk/blender/build_files/cmake/cmake_static_check_sparse.py	2011-09-22 03:14:50 UTC (rev 40443)
+++ trunk/blender/build_files/cmake/cmake_static_check_sparse.py	2011-09-22 04:41:12 UTC (rev 40444)
@@ -52,15 +52,21 @@
 
         check_commands.append((c, cmd))
 
-    for i, (c, cmd) in enumerate(check_commands):
+    def my_process(i, c, cmd):
         percent = 100.0 * (i / (len(check_commands) - 1))
         percent_str = "[" + ("%.2f]" % percent).rjust(7) + " %:"
 
+        sys.stdout.flush()
         sys.stdout.write("%s %s\n" % (percent_str, c))
-        sys.stdout.flush()
 
-        process = subprocess.Popen(cmd)
-        process.wait()
+        return subprocess.Popen(cmd)
 
+    process_functions = []
+    for i, (c, cmd) in enumerate(check_commands):
+        process_functions.append((my_process, (i, c, cmd)))
+
+    project_source_info.queue_processes(process_functions)
+
+
 if __name__ == "__main__":
     main()

Modified: trunk/blender/build_files/cmake/cmake_static_check_splint.py
===================================================================
--- trunk/blender/build_files/cmake/cmake_static_check_splint.py	2011-09-22 03:14:50 UTC (rev 40443)
+++ trunk/blender/build_files/cmake/cmake_static_check_splint.py	2011-09-22 04:41:12 UTC (rev 40444)
@@ -84,15 +84,21 @@
 
         check_commands.append((c, cmd))
 
-    for i, (c, cmd) in enumerate(check_commands):
+    def my_process(i, c, cmd):
         percent = 100.0 * (i / (len(check_commands) - 1))
         percent_str = "[" + ("%.2f]" % percent).rjust(7) + " %:"
 
         sys.stdout.write("%s %s\n" % (percent_str, c))
         sys.stdout.flush()
 
-        process = subprocess.Popen(cmd)
-        process.wait()
+        return subprocess.Popen(cmd)
 
+    process_functions = []
+    for i, (c, cmd) in enumerate(check_commands):
+        process_functions.append((my_process, (i, c, cmd)))
+
+    project_source_info.queue_processes(process_functions)
+
+
 if __name__ == "__main__":
     main()

Modified: trunk/blender/build_files/cmake/project_source_info.py
===================================================================
--- trunk/blender/build_files/cmake/project_source_info.py	2011-09-22 03:14:50 UTC (rev 40443)
+++ trunk/blender/build_files/cmake/project_source_info.py	2011-09-22 04:41:12 UTC (rev 40444)
@@ -148,6 +148,48 @@
     return source
 
 
+# could be moved elsewhere!, this just happens to be used by scripts that also
+# use this module.
+def queue_processes(process_funcs, job_total=-1):
+    """ Takes a list of function arg pairs, each function must return a process
+    """
+    import sys
+
+    if job_total == -1:
+        import multiprocessing
+        job_total = multiprocessing.cpu_count()
+        del multiprocessing
+
+    if job_total == 1:
+        import os
+        import sys
+        for func, args in process_funcs:
+            sys.stdout.flush()
+            sys.stderr.flush()
+
+            process = func(*args)
+            process.wait()
+    else:
+        import time
+        import subprocess
+
+        processes = []
+        for func, args in process_funcs:
+            # wait until a thread is free
+            while 1:
+                processes[:] = [p for p in processes if p.poll() is None]
+     
+                if len(processes) <= job_total:
+                    break
+                else:
+                    time.sleep(0.1)
+
+            sys.stdout.flush()
+            sys.stderr.flush()
+
+            processes.append(func(*args))
+
+
 def main():
     if not os.path.exists(join(CMAKE_DIR, "CMakeCache.txt")):
         print("This script must run from the cmake build dir")