[Durian-svn] [2722] Bugfixes and tweaks for new farm master.
brecht
institute at blender.org
Fri Apr 30 17:39:19 CEST 2010
Revision: 2722
https://blenderinstitute.dyndns.org/durian-svn/?do=log&project=durian&path=/&rev=2722
Author: brecht
Date: 2010-04-30 17:39:19 +0200 (Fri, 30 Apr 2010)
Log Message:
-----------
Bugfixes and tweaks for new farm master.
Modified Paths:
--------------
frm/master_ui.py
frm/new_blender_setup.py
frm/new_master.py
frm/new_node_update_frames.sh
Modified: frm/master_ui.py
===================================================================
--- frm/master_ui.py 2010-04-30 15:33:10 UTC (rev 2721)
+++ frm/master_ui.py 2010-04-30 15:39:19 UTC (rev 2722)
@@ -17,7 +17,7 @@
JOBS_FILE = os.path.join(FARM_DIR, "jobs.pkl")
SLAVES_FILE = os.path.join(FARM_DIR, "slaves.pkl")
REPO_PATH = "/media/data/svnroot/durian"
-HTTPD_IP = "127.0.0.1" # 192.168.1.14"
+HTTPD_IP = "192.168.1.14"
HTTPD_PORT = 8333
RESTART = False
GENERATE_AVI = False
@@ -274,23 +274,26 @@
else:
output("""<button onclick='request("/exec", "%s")'>%s</button>""" % (function, name))
- def return_file(path):
- f = open(path, 'rb')
-
+ def return_file(path, content_type):
self.send_response(http.client.OK)
- self.send_header("Content-type", "text/css")
+ self.send_header("Content-type", content_type)
self.end_headers()
- shutil.copyfileobj(f, self.wfile)
+ try:
+ f = open(path, 'rb')
+ shutil.copyfileobj(f, self.wfile)
+ f.close()
+ except IOError:
+ pass
- f.close()
-
if self.path == '/master_ui.css':
- return_file('master_ui.css')
+ return_file('master_ui.css', 'text/css')
elif self.path == '/master_ui.js':
- return_file('master_ui.js')
+ return_file('master_ui.js', 'text/javascript')
elif self.path == '/preview.png':
- return_file(os.path.join(FARM_DIR, 'slideshow/preview_small00001.png'))
+ return_file(os.path.join(FARM_DIR, 'slideshow/preview_small00001.png'), 'image/png')
+ elif self.path.startswith("/logs"):
+ return_file(os.path.join(FARM_DIR, self.path), 'text/plain')
else:
# headers
self.send_response(http.client.OK)
@@ -337,7 +340,7 @@
logs, revision = latest_svn_info()
output("<form method='post' action='/job_add'>\n")
- output("<input name='id' value='/d/pro/props/peach.blend' size='50'/>\n")
+ output("<input name='id' value='pro/props/peach.blend' size='50'/>\n")
output("<input name='revision' value='%s' size='10'/>\n" % (revision,))
output("<select name='quality'>\n")
for option in Job.quality_types:
@@ -367,13 +370,14 @@
# slaves
section("Slaves")
- table_begin("Name", "IP", "Status", "Enabled", "")
+ table_begin("Name", "IP", "Status", "Log", "Enabled", "")
for slave in SLAVES:
output("<tr>\n")
output("<td>" + slave.id + "</td>\n")
output("<td>" + slave.ip + "</td>\n")
output("<td>" + slave.status + "</td>\n")
+ output("<td><a href='/logs/%s.log'>view</a></td>\n" % (slave.ip,))
output("<td>")
checkbox('slave_set_enabled', slave.id, slave.enabled)
output("</td>\n")
@@ -418,8 +422,11 @@
Slave.load()
Job.load()
-HTTPD = http.server.HTTPServer((HTTPD_IP, HTTPD_PORT), HHandler)
+class MasterHTTPServer(http.server.HTTPServer):
+ allow_reuse_address = True
+HTTPD = MasterHTTPServer((HTTPD_IP, HTTPD_PORT), HHandler)
+
def update(timeout):
HTTPD.timeout = timeout
HTTPD.handle_request()
Modified: frm/new_blender_setup.py
===================================================================
--- frm/new_blender_setup.py 2010-04-30 15:33:10 UTC (rev 2721)
+++ frm/new_blender_setup.py 2010-04-30 15:39:19 UTC (rev 2722)
@@ -99,9 +99,9 @@
rd.threads = 12
'''
-render_frame = sys.argv[-1]
-scene.current_frame_start = render_frame
-scene.current_frame_end = render_frame
+render_frame = int(sys.argv[-1])
+scene.frame_start = render_frame
+scene.frame_end = render_frame
print("rendering frame %d" % (render_frame))
bpy.ops.render.render(animation=True)
Modified: frm/new_master.py
===================================================================
--- frm/new_master.py 2010-04-30 15:33:10 UTC (rev 2721)
+++ frm/new_master.py 2010-04-30 15:39:19 UTC (rev 2722)
@@ -7,9 +7,9 @@
# * re-enable svn update
# * backup old frames before starting a new job
# * remove .exrs to restart job
-# * find out why address already in use error happens
# * fix multiple nodes updating movie files
# * rsync is not killed currently
+# * Address already in use on kill
import os
import time
@@ -110,7 +110,10 @@
# update status
if image_done == len(images):
- job.status = "Done"
+ if image_done == 0:
+ job.status = "No Frames"
+ else:
+ job.status = "Done"
elif not job.enabled:
job.status = "Disabled"
else:
@@ -136,7 +139,6 @@
if job.enabled:
sorted_jobs.append(job)
- sorted_jobs = master_ui.JOBS
sorted_jobs.sort(key=lambda job: job.sortkey())
# get unfinished frame from sorted jobs
@@ -166,6 +168,12 @@
def is_busy(ip):
return os.path.exists(log_file(ip, ext="busy"))
+def touch_busy(ip):
+ os.system("echo \"Sending command\" > %s" % log_file(ip, ext="busy"))
+
+def clear_busy(ip):
+ os.system("rm %s" % log_file(ip, ext="busy"))
+
def available_slaves():
slaves = set()
for slave in master_ui.SLAVES:
@@ -175,21 +183,24 @@
def update_clear(ip, job, frame):
+ touch_busy(ip)
remote_command(ip, os.path.join(FARM_DIR, "node_update_clear.sh"))
def update_movie(ip, job, frame):
# do local svn update
- # XXX disabled to avoid interrupting
+ # XXX disabled to avoid interrupting
# os.system("svn revert -R %s" % os.path.join(FARM_DIR, "pro"))
# os.system("svn up -r%s %s" % (job.revision, os.path.join(FARM_DIR, "pro")))
+ touch_busy(ip)
# do remote rsync
remote_command(ip, os.path.join(FARM_DIR, "node_update_movie.sh"))
def update_frames(ip, job, frame):
# print("render jobs")
+ touch_busy(ip)
FARM_DIR_LOCAL = FARM_DIR.replace("/shared/software/durian_farm", "/d") # HACK
- remote_command(ip, os.path.join(FARM_DIR, "new_node_update_frames.sh") + " " + os.path.join(FARM_DIR_LOCAL, job.id))
+ remote_command(ip, os.path.join(FARM_DIR, "new_node_update_frames.sh") + " " + os.path.join(FARM_DIR_LOCAL, job.id) + " " + str(frame))
def stage_in():
@@ -204,7 +215,9 @@
# wait and watch for new jobs to be added, once they are, carry them out
for slave in master_ui.SLAVES:
- update_clear(slave.ip, None, 0)
+ clear_busy(slave.ip)
+ if slave.enabled:
+ update_clear(slave.ip, None, 0)
# states are tuples of (job, frame, update func nr)
slaves_state = {}
@@ -221,6 +234,8 @@
kill_non_critical = jobs_status_update()
master_ui.update(0.01)
+ changed = False
+
# add/remove available slaves
slaves = available_slaves()
@@ -228,12 +243,14 @@
# new systems
print("Adding:", slave.ip)
slaves_state[slave.ip] = (None, 0, 0)
+ changed = True
for slave in sorted(slaves_prev - slaves):
# removed systems
print("Removing:", slave.ip)
update_clear(slave.ip, None, 0)
del slaves_state[slave.ip]
+ changed = True
for slave in slaves:
# retrieve state
@@ -243,8 +260,10 @@
# priority, and if so, stop it so other jobs can be started
if slave_job:
if not slave_job.enabled or (slave_job.priority != "Critical" and kill_non_critical):
+ print("Cancelling job %s on %s" % (slave_job.id, slave.ip))
update_clear(slave.ip, None, 0)
slaves_state[slave.ip] = (None, 0, 0)
+ changed = True
# if slave is busy, do nothing
if is_busy(slave.ip):
@@ -257,9 +276,10 @@
slave_job = None
slave_frame = 0
slave_func_nr = 0
+ changed = True
# find a new job
- if slave_func_nr == 0:
+ if slave.enabled and slave_func_nr == 0:
slave_job, slave_frame = job_find(slaves_state)
if slave_job:
@@ -268,6 +288,7 @@
print(func.__name__, slave_func_nr, slave_job.id, slave_frame)
func(slave.ip, slave_job, slave_frame)
slave_func_nr += 1
+ changed = True
# update state
slaves_state[slave.ip] = (slave_job, slave_frame, slave_func_nr)
@@ -278,9 +299,12 @@
slaves_prev = set(slaves)
slaves_status_update()
- print("sleeping 8...")
- master_ui.update(8)
+ if changed: sleeptime = 1.0
+ else: sleeptime = 8.0
+ print("sleeping %.1f..." % (sleeptime,))
+ master_ui.update(sleeptime)
+
print("All Done")
def clear_all():
Modified: frm/new_node_update_frames.sh
===================================================================
--- frm/new_node_update_frames.sh 2010-04-30 15:33:10 UTC (rev 2721)
+++ frm/new_node_update_frames.sh 2010-04-30 15:39:19 UTC (rev 2722)
@@ -19,7 +19,7 @@
FRAME=$2
# get info on the file we're rendering
-echo "Rendering:"$BLEND":"$FRAME > $BUSY
+echo "Rendering "$BLEND":"$FRAME > $BUSY
# -a doesnt work. TODO, find out why, render with operator until then.
nice -n 19 $PREFIX_SRC/blender_farm.sh -b -noaudio $BLEND -P $FARM_DIR/new_blender_setup.py -- $FRAME
More information about the Durian-svn
mailing list