[Durian-svn] [3865] various updates from brecht and myself, these were not being committed because on an svn version problem on the server .
campbell
institute at blender.org
Wed May 26 08:18:12 CEST 2010
Revision: 3865
https://blenderinstitute.dyndns.org/durian-svn/?do=log&project=durian&path=/&rev=3865
Author: campbell
Date: 2010-05-26 08:18:12 +0200 (Wed, 26 May 2010)
Log Message:
-----------
various updates from brecht and myself, these were not being committed because on an svn version problem on the server.
Modified Paths:
--------------
frm/blender_exr_to_avi.py
frm/ip.sh
frm/ips.py
frm/master_avi_gen.py
frm/master_ui.css
frm/master_ui.py
frm/new_master.py
frm/new_node_update_movie.sh
frm/slideshow/preview_images_update.py
frm/slideshow/slideshow.sh
frm/ssh_all_farm.py
Modified: frm/blender_exr_to_avi.py
===================================================================
--- frm/blender_exr_to_avi.py 2010-05-26 06:15:35 UTC (rev 3864)
+++ frm/blender_exr_to_avi.py 2010-05-26 06:18:12 UTC (rev 3865)
@@ -1,12 +1,26 @@
import sys
-image_dir = sys.argv[-1]
+import os
+
+sys.path.append("/shared/software/durian_farm/")
+
+import new_blend_2_frames
+
+image_dir = sys.argv[-2]
+blendfile = sys.argv[-1]
# image_dir = "/shared/software/durian_farm/frames/03.4f/"
if not image_dir.endswith("/"):
image_dir += "/"
-import os
-files = [{"name": f} for f in sorted(os.listdir(image_dir)) if f.endswith(".exr")]
+frames = new_blend_2_frames.blend_2_frames(blendfile)
+files = []
+
+print(frames, blendfile)
+
+for f_nr, f_name in frames:
+ f_dir, f_fname = os.path.split(f_name)
+ files += [{"name": f_fname}]
+
bpy.ops.sequencer.image_strip_add(path=image_dir, filename="test.blend", directory=image_dir, filter_blender=False, filter_image=True, filter_movie=False, name="", frame_start=1, channel=1, replace_sel=True, files=files)
# render
Modified: frm/ip.sh
===================================================================
--- frm/ip.sh 2010-05-26 06:15:35 UTC (rev 3864)
+++ frm/ip.sh 2010-05-26 06:18:12 UTC (rev 3865)
@@ -1 +1 @@
-ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $1}'
+ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 | head -n 1 | awk '{ print $1}'
Modified: frm/ips.py
===================================================================
--- frm/ips.py 2010-05-26 06:15:35 UTC (rev 3864)
+++ frm/ips.py 2010-05-26 06:18:12 UTC (rev 3865)
@@ -16,9 +16,9 @@
ips_all += ["192.168.1.27"] # nathan
'''
-ips_all += ["192.168.1.201"]
-ips_all += ["192.168.1.202"]
-ips_all += ["192.168.1.203"]
-ips_all += ["192.168.1.204"]
+ips_all += ["192.168.1.205"]
+ips_all += ["192.168.1.206"]
+ips_all += ["192.168.1.207"]
+ips_all += ["192.168.1.208"]
-ips_all_farm = ips_all[:]
\ No newline at end of file
+ips_all_farm = ips_all[:]
Modified: frm/master_avi_gen.py
===================================================================
--- frm/master_avi_gen.py 2010-05-26 06:15:35 UTC (rev 3864)
+++ frm/master_avi_gen.py 2010-05-26 06:18:12 UTC (rev 3865)
@@ -25,21 +25,16 @@
def generate(ip, user="guest"):
frame_dir = os.path.join(FARM_DIR, "frames")
- frames = os.listdir(frame_dir)
- frames.sort()
+ frame_dirs = os.listdir(frame_dir)
+ frame_dirs.sort()
commit_avis = []
- for fdir in frames:
+ for fdir in frame_dirs:
image_dir = os.path.join(frame_dir, fdir)
if not os.path.isdir(image_dir):
continue
- files = os.listdir(image_dir)
-
- exrs = [f for f in files if f.endswith(".exr")]
- avis = [f for f in files if f == fdir + ".avi"]
-
# find number of frames
blendfile = find_file(os.path.join(FARM_DIR, "pro"), fdir + ".blend")
if not blendfile:
@@ -48,16 +43,31 @@
#os.system("svn up %s > /dev/null" % blendfile)
frames = new_blend_2_frames.blend_2_frames(blendfile)
+ # find avis and exrs
+ files = os.listdir(image_dir)
+
+ exrs = []
+ avi = None
+ if fdir + ".avi" in files:
+ avi = fdir + ".avi"
+
+ for frame_nr, frame_path in frames:
+ f_dir, f_fname = os.path.split(frame_path)
+ if f_fname in files:
+ exrs += [f_fname]
+
+ #if len(exrs) != len([f for f in files if f.endswith(".exr")]):
+ # print("frames mismatch %s %d vs %d" % (fdir, len(exrs), len([f for f in files if f.endswith(".exr")])))
+
skip = False
if len(exrs) == 0 or len(exrs) != len(frames):
- print("frames not complete: %s" % image_dir)
+ print("frames not complete (%d/%d): %s" % (len(exrs), len(frames), image_dir))
skip = True
- elif avis:
+ elif avi:
exr_date = max([file_date(os.path.join(image_dir, f)) for f in exrs])
- avi_date = max([file_date(os.path.join(image_dir, f)) for f in avis])
+ avi_date = file_date(os.path.join(image_dir, avi))
- for avi in avis:
- commit_avis += [(image_dir, avi)]
+ commit_avis += [(image_dir, avi)]
# avi is newer
if avi_date > exr_date:
@@ -65,15 +75,16 @@
skip = True
if not skip:
- cmd = "ssh -p 22 %s@%s '%s -b -P %s -- %s'" % (user, ip, BIN, SCRIPT, image_dir)
+ cmd = "ssh -p 22 %s@%s '%s -b -P %s -- %s %s'" % (user, ip, BIN, SCRIPT, image_dir, blendfile)
print(cmd)
try:
os.system(cmd)
except:
- for avi in avis:
- os.system("rm %s" % os.path.join(image_dir, avi))
+ os.system("rm %s" % os.path.join(image_dir, avi))
+ os.system("svn up " + os.path.join(FARM_DIR, "ani/render"))
+
commit_files = ""
for dir, avi in commit_avis:
from_file = os.path.join(dir, avi)
Modified: frm/master_ui.css
===================================================================
--- frm/master_ui.css 2010-05-26 06:15:35 UTC (rev 3864)
+++ frm/master_ui.css 2010-05-26 06:18:12 UTC (rev 3865)
@@ -1,7 +1,7 @@
body {
background-color:#000;
color: #ddd;
- font-size:12px;
+ font-size:11px;
font-family: "Lucida Sans","Lucida Sans Unicode","Lucida Grande",Lucida,sans-serif;
}
a {
@@ -124,7 +124,7 @@
}
tr {
- font-size: 14px !important;
+ font-size: 12px !important;
background-color: #111;
}
Modified: frm/master_ui.py
===================================================================
--- frm/master_ui.py 2010-05-26 06:15:35 UTC (rev 3864)
+++ frm/master_ui.py 2010-05-26 06:18:12 UTC (rev 3865)
@@ -2,6 +2,7 @@
import cgi
import datetime
+import fcntl
import http.server
import os
import os.path
@@ -14,7 +15,6 @@
from render_dirs import FARM_DIR
from render_dirs import GEN_AVI_LOG
from render_dirs import GEN_AVI_BUSY
-FARM_DIR = "/media/data/durian/frm/"
TOTAL_PROGRESS = "No statistics yet."
JOBS = []
@@ -22,9 +22,8 @@
JOBS_FILE = os.path.join(FARM_DIR, "jobs.pkl")
SLAVES_FILE = os.path.join(FARM_DIR, "slaves.pkl")
REPO_PATH = "/media/data/svnroot/durian"
-#HTTPD_IP = "192.168.1.14"
-HTTPD_IP = "127.0.0.1"
-HTTPD_PORT = 8334
+HTTPD_IP = "192.168.3.14"
+HTTPD_PORT = 8333
RESTART = False
GENERATE_AVI = False
STOP_AVI = False
@@ -64,6 +63,7 @@
self.revision = revision
self.quality = quality
self.enabled = False
+ self.stats = ""
@staticmethod
def find(id):
@@ -140,6 +140,15 @@
JOBS.remove(Job.find(id))
Job.dump()
+def job_set_stats(job):
+ import exr_header_collate
+ render_times = exr_header_collate.render_times(job.id)
+ if render_times:
+ time_avg, time_min, time_max = render_times
+ time_avg = str(datetime.timedelta(0, round(time_avg)))
+ time_max = str(datetime.timedelta(0, round(time_max)))
+ job.stats = time_avg + " avg, " + time_max + " max"
+
############################ Slaves ########################
class Slave:
@@ -344,7 +353,7 @@
# jobs
section("Jobs")
- table_begin("", "File", "Revision", "Quality", "Status", "Progress", "Priority", "")
+ table_begin("", "File", "Revision", "Quality", "Status", "Progress", "Stats", "Priority", "")
for job in JOBS:
output("<tr>\n")
@@ -363,6 +372,7 @@
else:
output("<td>" + job.status + "</td>\n")
output("<td class='centered'>" + job.progress + "</td>\n")
+ output("<td class='centered'>" + job.stats + "</td>\n")
output("<td>")
dropdown('job_set_priority', job.id, job.priority, Job.priority_types)
output("</td>\n")
@@ -378,7 +388,7 @@
logs, revision = latest_svn_info()
output("<form method='post' action='/job_add'>\n")
- output("<input name='id' value='pro/scenes/' size='50'/>\n")
+ output("<input name='id' value='' size='50'/>\n")
output("<input name='revision' value='%s' size='10'/>\n" % (revision,))
output("<select name='quality'>\n")
for option in Job.quality_types:
@@ -491,11 +501,28 @@
Slave.load()
Job.load()
+# temporariry compat code to add stats for existing jobs
+for job in JOBS:
+ if not "stats" in dir(job):
+ job_set_stats(job)
+ if len(job.id) > 500:
+ job_remove(job.id)
+ if job.id.find("href") != -1:
+ job_remove(job.id)
+
class MasterHTTPServer(http.server.HTTPServer):
allow_reuse_address = True
+ # child processes inherit sockets, and prevent us from restarting server
+ # on same port, this flag ensures the socket is closed when the parent
+ # process is killed
+ def socket_inherit_fix(self):
+ f = self.socket.fileno()
+ fcntl.fcntl(f, fcntl.F_SETFD, fcntl.fcntl(f, fcntl.F_GETFD)|fcntl.FD_CLOEXEC)
+
print("starting httpd at %s:%d" % (HTTPD_IP, HTTPD_PORT))
HTTPD = MasterHTTPServer((HTTPD_IP, HTTPD_PORT), HHandler)
+HTTPD.socket_inherit_fix()
def update(timeout):
starttime = time.time()
Modified: frm/new_master.py
===================================================================
--- frm/new_master.py 2010-05-26 06:15:35 UTC (rev 3864)
+++ frm/new_master.py 2010-05-26 06:18:12 UTC (rev 3865)
@@ -123,7 +123,7 @@
# update progress
if image_tot == 0: percentage = 0.0
else: percentage = float(image_done)/float(image_tot)
- job.progress = "%.2f%% (%d/%d)" % (100*percentage, image_done, image_tot)
+ job.progress = "%.0f%% (%d/%d)" % (100*percentage, image_done, image_tot)
# update total progress
if total_tot == 0: percentage = 0.0
@@ -325,6 +325,9 @@
# if slave is finished, go back to stage zero
if slave_func_nr >= len(update_funcs):
+ if slave_job:
+ master_ui.job_set_stats(slave_job)
+
print("finished: %s" % slave.ip)
slave_job = None
slave_frame = 0
@@ -339,16 +342,16 @@
# call next update function
func = update_funcs[slave_func_nr]
- # only two computers allowed to update svn at
@@ Diff output truncated at 10240 characters. @@
More information about the Durian-svn
mailing list