[Durian-svn] [2755] Move away old frames on start, use --force for svn up and allow
brecht
institute at blender.org
Mon May 3 12:24:24 CEST 2010
Revision: 2755
https://blenderinstitute.dyndns.org/durian-svn/?do=log&project=durian&path=/&rev=2755
Author: brecht
Date: 2010-05-03 12:24:24 +0200 (Mon, 03 May 2010)
Log Message:
-----------
Move away old frames on start, use --force for svn up and allow
it to be killed, previously would just keep updating.
Modified Paths:
--------------
frm/master_ui.py
frm/new_master.py
frm/new_node_update_frames.sh
frm/new_node_update_movie.sh
frm/slideshow/preview_images_update.py
frm/slideshow/slideshow.sh
Added Paths:
-----------
frm/new_node_command.sh
frm/new_node_update_clear.sh
Modified: frm/master_ui.py
===================================================================
--- frm/master_ui.py 2010-05-03 10:10:20 UTC (rev 2754)
+++ frm/master_ui.py 2010-05-03 10:24:24 UTC (rev 2755)
@@ -107,11 +107,6 @@
Job.find(id).quality = quality
Job.dump()
-# XXX not implemented yet
-#def job_clear(params):
-# Job.find(id).progress = "0/0"
-# Job.dump()
-
def job_add(id, revision, quality):
# master loop will find the new job when refreshes it's jobs list in
# the outer loop, so actually starting to render may take a while
@@ -119,9 +114,13 @@
if job.id == id: # only one job per file supported
return
- JOBS.append(Job(id, revision, quality))
+ job = Job(id, revision, quality)
+ JOBS.append(job)
Job.dump()
+ import new_master
+ new_master.move_old_frames(job)
+
def job_remove(id):
# disabling first ensures that any slaves rendering this stop the job
job_set_enabled(id, False)
@@ -311,17 +310,6 @@
title("Durian Farm")
- # instructions
- section("Temporary Instructions")
- output("<ul>")
- output("<li>Don't submit the same files as on the other farm here, it will mess up.</li>")
- output("<li>Don't add the render nodes as slaves, will not work correct.</li>")
- output("<li>Rerendering scenes that already have frames need those manually moved or removed.</li>")
- output("<li>Only works correct with relative paths, otherwise may use older/newer files from your checkout.</li>")
- output("</ul>")
-
- output("<br/>\n")
-
# jobs
section("Jobs")
table_begin("File", "Revision", "Quality", "Status", "Progress", "Priority", "Enabled", "")
@@ -340,7 +328,6 @@
checkbox('job_set_enabled', job.id, job.enabled)
output("</td>\n")
output("<td>")
- #action('job_clear(\\"%s\\")' % job.id, "Clear", "Are you sure you want to remove all rendered frames?")
action('job_remove(\\"%s\\")' % job.id, "X", "Are you sure you want to remove this job?")
output("</td>\n")
output("</tr>")
@@ -443,6 +430,9 @@
HTTPD.timeout = timeout
HTTPD.handle_request()
+def close():
+ HTTPD.server_close()
+
if __name__ == "__main__":
HTTPD.serve_forever()
Modified: frm/new_master.py
===================================================================
--- frm/new_master.py 2010-05-03 10:10:20 UTC (rev 2754)
+++ frm/new_master.py 2010-05-03 10:24:24 UTC (rev 2755)
@@ -3,10 +3,6 @@
# TODO
# * auto-generate avis after job is done, non-blocking
# * detect crashes so it does not get stuck on these frames
-# * svn updates on slaves are unreliable...
-# * backup old frames before starting a new job
-# * remove .exrs to restart job
-# * svn update is not killed currently
# * httpd address already in use happens sometimes on kill
import os
@@ -163,6 +159,38 @@
# nothing to do
return None, 0
+def move_old_frames(job):
+ def make_old(filename):
+ import shutil
+
+ dirname = os.path.dirname(filename)
+ basename = os.path.basename(filename)
+
+ old_dirname = os.path.join(dirname, "old")
+
+ # make sure the target dir exists
+ try:
+ os.mkdir(old_dirname)
+ except:
+ pass
+ os.system("chmod 777 " + old_dirname)
+
+ old_filename = os.path.join(old_dirname, basename)
+
+ try:
+ shutil.move(filename, old_filename)
+ except:
+ os.system("rm -f " + filename)
+
+ blendfile_abs = os.path.join(FARM_DIR, job.id)
+
+ if os.path.exists(blendfile_abs):
+ images = new_blend_2_frames.blend_2_frames(blendfile_abs)
+
+ for frame, image in images:
+ if os.path.exists(image):
+ make_old(image)
+
def is_busy(ip):
return os.path.exists(log_file(ip, ext="busy"))
@@ -183,7 +211,7 @@
def update_clear(ip, job, frame):
touch_busy(ip)
- remote_command(ip, os.path.join(FARM_DIR, "node_update_clear.sh"))
+ remote_command(ip, os.path.join(FARM_DIR, "new_node_update_clear.sh"))
def update_movie(ip, job, frame):
# do local svn update
@@ -322,6 +350,7 @@
#clear_all()
# while 1:
# time.sleep(2)
+ master_ui.close()
if __name__ == "__main__":
main()
Added: frm/new_node_command.sh
===================================================================
--- frm/new_node_command.sh (rev 0)
+++ frm/new_node_command.sh 2010-05-03 10:24:24 UTC (rev 2755)
@@ -0,0 +1,4 @@
+#!/bin/bash
+# wrapper so we can do a simple killall to stop any a command
+$@
+
Copied: frm/new_node_update_clear.sh (from rev 2699, frm/node_update_clear.sh)
===================================================================
--- frm/new_node_update_clear.sh (rev 0)
+++ frm/new_node_update_clear.sh 2010-05-03 10:24:24 UTC (rev 2755)
@@ -0,0 +1,16 @@
+#!/bin/bash
+PATH=$PATH:$FARM_DIR
+export IP=`ip.sh`
+BUSY=$FARM_DIR/logs/$IP.busy
+
+echo "Cleaning..." > $BUSY
+
+# do this first to signal that new jobs shouldnt spawn once blender is killed
+rm $BUSY
+
+pkill -P $(pgrep -f new_node_command.sh)
+
+rm -rf /tmp/durian_farm
+rm -rf /tmp/*.exr
+rm -rf /tmp/*.blend
+
Property changes on: frm/new_node_update_clear.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mergeinfo
+
Modified: frm/new_node_update_frames.sh
===================================================================
--- frm/new_node_update_frames.sh 2010-05-03 10:10:20 UTC (rev 2754)
+++ frm/new_node_update_frames.sh 2010-05-03 10:24:24 UTC (rev 2755)
@@ -22,6 +22,6 @@
echo "Rendering "$BLEND":"$FRAME > $BUSY
# -a doesnt work. TODO, find out why, render with operator until then.
-nice -n 19 $PREFIX_SRC/blender_farm.sh -b -noaudio $BLEND -P $FARM_DIR/new_blender_setup.py -- $FRAME
+$FARM_DIR/new_node_command.sh nice -n 19 $PREFIX_SRC/blender_farm.sh -b -noaudio $BLEND -P $FARM_DIR/new_blender_setup.py -- $FRAME
rm $BUSY
Modified: frm/new_node_update_movie.sh
===================================================================
--- frm/new_node_update_movie.sh 2010-05-03 10:10:20 UTC (rev 2754)
+++ frm/new_node_update_movie.sh 2010-05-03 10:24:24 UTC (rev 2755)
@@ -10,10 +10,10 @@
if [ -d $DIR ]; then
cd $DIR/pro
svn cleanup
- svn up -r$1
+ $FARM_DIR/new_node_command.sh svn up --force -r$1
else
mkdir $DIR
- svn co svn://biserver/durian/pro $DIR/pro
+ $FARM_DIR/new_node_command.sh svn co --force svn://biserver/durian/pro $DIR/pro
fi
rm $BUSY
Modified: frm/slideshow/preview_images_update.py
===================================================================
--- frm/slideshow/preview_images_update.py 2010-05-03 10:10:20 UTC (rev 2754)
+++ frm/slideshow/preview_images_update.py 2010-05-03 10:24:24 UTC (rev 2755)
@@ -7,39 +7,44 @@
CONVERT = "/shared/software/durian_farm/slideshow/exr_to_png.sh %s %s %d > /dev/null 2> /dev/null"
PNG_FILE = "/shared/software/durian_farm/slideshow/preview"
NUM_IMAGES = 2
+NUM_SMALL_IMAGES = 2
SLEEP = 120.0
while 1:
- print "scanning for exr files"
+ print "scanning for exr files"
- # find images to display
- exr_files = []
+ # find images to display
+ exr_files = []
- for root, dirs, files in os.walk(DIR):
- print root
+ for root, dirs, files in os.walk(DIR):
+ print root
- for file in files:
- if file.endswith('.exr'):
- name = os.path.join(root, file)
- st = os.stat(name)
+ for file in files:
+ if file.endswith('.exr'):
+ name = os.path.join(root, file)
+ st = os.stat(name)
- if st.st_size > 10:
- exr_files += [(name, st.st_mtime)]
+ if st.st_size > 10:
+ exr_files += [(name, st.st_mtime)]
- exr_files.sort(key=lambda pair: pair[1])
- exr_files = exr_files[-NUM_IMAGES:]
- exr_files.reverse()
+ exr_files.sort(key=lambda pair: pair[1])
+ exr_files = exr_files[-NUM_IMAGES:]
+ exr_files.reverse()
- # convert images
- for i in range(0, NUM_IMAGES):
- name, mtime = exr_files[i]
- print "converting", name
- command = CONVERT % (name, PNG_FILE + str(i), 100)
- os.system(command)
- command = CONVERT % (name, PNG_FILE + "_small" + str(i), 30)
- os.system(command)
+ # convert images
+ for i in range(0, NUM_IMAGES):
+ name, mtime = exr_files[i]
+ print "converting", name
+ command = CONVERT % (name, PNG_FILE + "_big" + str(i), 100)
+ os.system(command)
- # sleep a while until the next up
- print "sleeping for", str(SLEEP/60.0), "minutes"
- time.sleep(SLEEP)
+ for i in range(0, NUM_SMALL_IMAGES):
+ name, mtime = exr_files[i]
+ print "converting small", name
+ command = CONVERT % (name, PNG_FILE + "_small" + str(i), 30)
+ os.system(command)
+ # sleep a while until the next up
+ print "sleeping for", str(SLEEP/60.0), "minutes"
+ time.sleep(SLEEP)
+
Modified: frm/slideshow/slideshow.sh
===================================================================
--- frm/slideshow/slideshow.sh 2010-05-03 10:10:20 UTC (rev 2754)
+++ frm/slideshow/slideshow.sh 2010-05-03 10:24:24 UTC (rev 2755)
@@ -1,2 +1,2 @@
#!/bin/bash
-mplayer -loop 0 -mf fps=0.0166 -vo xv,x11 -fixed-vo -fs mf:///shared/software/durian_farm/slideshow/preview*.png
+mplayer -loop 0 -mf fps=0.0166 -vo xv,x11 -fixed-vo -fs mf:///shared/software/durian_farm/slideshow/preview_big*.png
More information about the Durian-svn
mailing list