[Durian-svn] [3300] Assorted farm script updates that were not committed yet:
brecht
institute at blender.org
Thu May 13 12:31:36 CEST 2010
Revision: 3300
https://blenderinstitute.dyndns.org/durian-svn/?do=log&project=durian&path=/&rev=3300
Author: brecht
Date: 2010-05-13 12:31:36 +0200 (Thu, 13 May 2010)
Log Message:
-----------
Assorted farm script updates that were not committed yet:
* async avi generating, auto committing to svn, new filenames
* cpu optimizations for farm and preview image gen scripts
* some other minor updates
Modified Paths:
--------------
frm/blender_exr_to_avi.py
frm/master_avi_gen.py
frm/master_ui.py
frm/new_blender_setup.py
frm/new_master.py
frm/render_dirs.py
frm/slideshow/preview_images_update.py
frm/ssh_all_farm.py
Modified: frm/blender_exr_to_avi.py
===================================================================
--- frm/blender_exr_to_avi.py 2010-05-13 10:19:38 UTC (rev 3299)
+++ frm/blender_exr_to_avi.py 2010-05-13 10:31:36 UTC (rev 3300)
@@ -18,7 +18,6 @@
bpy.context.scene.render.resolution_y = 872
bpy.context.scene.render.file_format = 'AVI_JPEG'
bpy.context.scene.render.file_quality = 95
-bpy.context.scene.render.output_path = image_dir
+bpy.context.scene.render.output_path = os.path.join(image_dir, os.path.basename(image_dir[:-1])) + ".avi"
-
bpy.ops.render.render(animation=True)
Modified: frm/master_avi_gen.py
===================================================================
--- frm/master_avi_gen.py 2010-05-13 10:19:38 UTC (rev 3299)
+++ frm/master_avi_gen.py 2010-05-13 10:31:36 UTC (rev 3300)
@@ -2,6 +2,7 @@
import os
import time
+import new_blend_2_frames
from render_dirs import FARM_DIR
@@ -15,10 +16,20 @@
# year, month, day, hour, minute, second, weekday, yearday, daylight = now
return time.localtime(os.stat(path)[8])[0:6]
+def find_file(dir, filename):
+ for root, dirs, files in os.walk(dir):
+ if filename in files:
+ return os.path.join(root, filename)
+
+ return None
+
def generate(ip, user="guest"):
frame_dir = os.path.join(FARM_DIR, "frames")
frames = os.listdir(frame_dir)
+ frames.sort()
+ commit_avis = []
+
for fdir in frames:
image_dir = os.path.join(frame_dir, fdir)
if not os.path.isdir(image_dir):
@@ -27,27 +38,59 @@
files = os.listdir(image_dir)
exrs = [f for f in files if f.endswith(".exr")]
- avis = [f for f in files if f.endswith(".avi")]
+ avis = [f for f in files if f == fdir + ".avi"]
+ # find number of frames
+ blendfile = find_file(os.path.join(FARM_DIR, "pro"), fdir + ".blend")
+ if not blendfile:
+ continue
+
+ #os.system("svn up %s > /dev/null" % blendfile)
+ frames = new_blend_2_frames.blend_2_frames(blendfile)
+
skip = False
- if not exrs:
+ if len(exrs) == 0 or len(exrs) != len(frames):
+ print("frames not complete: %s" % image_dir)
skip = True
elif avis:
exr_date = max([file_date(os.path.join(image_dir, f)) for f in exrs])
avi_date = max([file_date(os.path.join(image_dir, f)) for f in avis])
+
+ for avi in avis:
+ commit_avis += [(image_dir, avi)]
# avi is newer
if avi_date > exr_date:
+ print("avi up to date: %s" % image_dir)
skip = True
if not skip:
cmd = "ssh -p 22 %s@%s '%s -b -P %s -- %s'" % (user, ip, BIN, SCRIPT, image_dir)
print(cmd)
- os.system(cmd)
- else:
- print("avi up to date: %s" % image_dir)
+ try:
+ os.system(cmd)
+ except:
+ for avi in avis:
+ os.system("rm %s" % os.path.join(image_dir, avi))
+ commit_files = ""
+ for dir, avi in commit_avis:
+ from_file = os.path.join(dir, avi)
+ to_file = os.path.join(FARM_DIR, "ani/render", avi)
+ if os.path.exists(to_file):
+ print(to_file, file_date(to_file))
+ if os.path.exists(from_file):
+ print(from_file, file_date(from_file))
+ if not os.path.exists(to_file) or (os.path.exists(from_file) and file_date(from_file) > file_date(to_file)):
+ print("copying " + from_file)
+ os.system("cp %s %s" % (from_file, to_file))
+ commit_files += " \"%s\"" % to_file
+ print("committing " + commit_files)
+ os.system("svn add " + commit_files)
+ #os.system("svn status " + commit_files)
+ os.system("svn commit -m \"Updated .avi's from the farm.\" " + commit_files)
+
if __name__ == '__main__':
- generate("192.168.1.12")
\ No newline at end of file
+ generate("192.168.1.12")
Modified: frm/master_ui.py
===================================================================
--- frm/master_ui.py 2010-05-13 10:19:38 UTC (rev 3299)
+++ frm/master_ui.py 2010-05-13 10:31:36 UTC (rev 3300)
@@ -12,7 +12,9 @@
import urllib
from render_dirs import FARM_DIR
-#FARM_DIR = "/media/data/durian/frm/"
+from render_dirs import GEN_AVI_LOG
+from render_dirs import GEN_AVI_BUSY
+FARM_DIR = "/media/data/durian/frm/"
TOTAL_PROGRESS = "No statistics yet."
JOBS = []
@@ -20,11 +22,12 @@
JOBS_FILE = os.path.join(FARM_DIR, "jobs.pkl")
SLAVES_FILE = os.path.join(FARM_DIR, "slaves.pkl")
REPO_PATH = "/media/data/svnroot/durian"
-HTTPD_IP = "192.168.1.14"
-#HTTPD_IP = "127.0.0.1"
-HTTPD_PORT = 8333
+#HTTPD_IP = "192.168.1.14"
+HTTPD_IP = "127.0.0.1"
+HTTPD_PORT = 8334
RESTART = False
GENERATE_AVI = False
+STOP_AVI = False
MOVE_OLD_FRAMES = None
def latest_svn_info():
@@ -119,9 +122,13 @@
if job.id == id: # only one job per file supported
return
+ command = "svn up -r %s %s" % (revision, os.path.join(FARM_DIR, id))
+ print(command)
+ os.system(command)
+
job = Job(id, revision, quality)
JOBS.append(job)
- JOBS.sort(key=lambda job: job.id)
+ JOBS.sort(key=lambda job: job.revision)
Job.dump()
if MOVE_OLD_FRAMES:
@@ -193,6 +200,10 @@
global GENERATE_AVI
GENERATE_AVI = True
+def master_stop_avi():
+ global STOP_AVI
+ STOP_AVI = True
+
def master_restart():
global RESTART
RESTART = True
@@ -450,16 +461,27 @@
output("<hr/>\n")
+ # avi's
+ section("Generate AVI's")
+
+ busyfile = os.path.join(FARM_DIR, GEN_AVI_BUSY)
+
+ if os.path.exists(busyfile) or GENERATE_AVI:
+ action('master_stop_avi()', "Stop", "")
+ output("... working ...")
+ else:
+ action('master_generate_avi()', "Start", "")
+
+ output(" <a href='/%s'>log</a>\n" % GEN_AVI_LOG)
+ output("<br/><br/>")
+
# render farm
section("Master Controls")
action('master_restart()', "Restart", "Are you sure?")
- action('master_generate_avi()', "Generate AVI's", "Are you sure? This will block access to this interface until done (sorry).")
output("<br/>\n")
if RESTART:
output("waiting to restart...<br/>")
- if GENERATE_AVI:
- output("waiting to generate avi's...<br/>")
# html end
output("</div></body></html>\n")
@@ -472,11 +494,17 @@
class MasterHTTPServer(http.server.HTTPServer):
allow_reuse_address = True
+print("starting httpd at %s:%d" % (HTTPD_IP, HTTPD_PORT))
HTTPD = MasterHTTPServer((HTTPD_IP, HTTPD_PORT), HHandler)
def update(timeout):
- HTTPD.timeout = timeout
- HTTPD.handle_request()
+ starttime = time.time()
+ left = timeout
+ while left >= 0.0:
+ print("httpd handle requests for %.2fs" % left)
+ HTTPD.timeout = left
+ HTTPD.handle_request()
+ left = timeout - (time.time() - starttime)
def close():
HTTPD.server_close()
Modified: frm/new_blender_setup.py
===================================================================
--- frm/new_blender_setup.py 2010-05-13 10:19:38 UTC (rev 3299)
+++ frm/new_blender_setup.py 2010-05-13 10:31:36 UTC (rev 3300)
@@ -4,8 +4,15 @@
import os
import sys
+try:
+ import finals_config
+except:
+ print("finals_config not found")
+ sys.exit(1)
+
render_frame = int(sys.argv[-3])
quality = sys.argv[-2] + " " + sys.argv[-1]
+print(quality)
def get_ip():
@@ -51,6 +58,7 @@
# file output
rd.file_format = 'OPEN_EXR'
rd.exr_half = True
+ rd.exr_preview = True
rd.output_path = "/shared/software/durian_farm/frames/%s/%s_######" % (fname, fname)
rd.use_file_extension = True
rd.use_placeholder = False
@@ -104,7 +112,7 @@
elif quality == "Simplified 1K":
rd.resolution_percentage = 50
- rd.use_textures = False
+ rd.use_textures = True
rd.use_raytracing = False
rd.use_sss = False
rd.use_shadows = False
@@ -113,6 +121,7 @@
rd.simplify_subdivision = 0
rd.simplify_child_particles = 0.1
rd.simplify_triangulate = True
+ rd.simplify_mipmap_levels = 5
elif quality == "Final 4K":
# is this resolution right?
rd.resolution_percentage = 200
@@ -126,6 +135,9 @@
print("rendering frame %d" % (render_frame))
+# TODO, 02.f.blend has this bug!
+scene_current.set_frame(scene_current.frame_current)
+
# TODO, some valid file test would be nice.
if not fpath.startswith("<memory"):
bpy.ops.render.render(animation=True)
Modified: frm/new_master.py
===================================================================
--- frm/new_master.py 2010-05-13 10:19:38 UTC (rev 3299)
+++ frm/new_master.py 2010-05-13 10:31:36 UTC (rev 3300)
@@ -6,11 +6,14 @@
# * link durian directory to durian_farm_svn on nodes
# * still using local svn to check for blend_2_frames
+import gc
import os
import time
import master_ui
import new_blend_2_frames
from render_dirs import FARM_DIR
+from render_dirs import GEN_AVI_LOG
+from render_dirs import GEN_AVI_BUSY
os.umask(777)
@@ -235,6 +238,19 @@
FARM_DIR_LOCAL = "/media/data/durian_farm_svn"
remote_command(ip, os.path.join(FARM_DIR, "new_node_update_frames.sh") + " " + os.path.join(FARM_DIR_LOCAL, job.id) + " " + str(frame) + " \"" + job.quality + "\"")
+def generate_avi():
+ busypath = os.path.join(FARM_DIR, GEN_AVI_BUSY)
+ logpath = os.path.join(FARM_DIR, GEN_AVI_LOG)
+
+ if not os.path.exists(busypath):
+ os.system("%s >> %s 2>&1 &" % (os.path.join(FARM_DIR, "new_gen_avi.py"), logpath))
+
+def stop_avi():
+ busypath = os.path.join(FARM_DIR, GEN_AVI_BUSY)
+
+ os.system("killall -9 new_gen_avi.py")
+ os.system("rm %s" % busypath)
+
def stage_in():
'''
@@ -259,13 +275,14 @@
slaves_prev = set()
while not master_ui.RESTART:
+ # avi generating controlled by the user
+ if master_ui.STOP_AVI:
+ stop_avi()
+ master_ui.STOP_AVI = False
if master_ui.GENERATE_AVI:
- # update avis when requested by user
+ generate_avi()
master_ui.GENERATE_AVI = False
- import master_avi_gen
- master_avi_gen.generate('192.168.1.201', user='root')
-
kill_non_critical = jobs_status_update()
@@ Diff output truncated at 10240 characters. @@
More information about the Durian-svn
mailing list