[Durian-svn] [2916] Updated farm UI from pablo and other assorted fixes.
brecht
institute at blender.org
Wed May 5 16:42:30 CEST 2010
Revision: 2916
https://blenderinstitute.dyndns.org/durian-svn/?do=log&project=durian&path=/&rev=2916
Author: brecht
Date: 2010-05-05 16:42:30 +0200 (Wed, 05 May 2010)
Log Message:
-----------
Updated farm UI from pablo and other assorted fixes.
Modified Paths:
--------------
frm/blend_2_frames.py
frm/master_ui.py
frm/new_blend_2_frames.py
frm/new_blender_setup.py
frm/new_master.py
frm/new_node_update_clear.sh
frm/new_node_update_movie.sh
frm/slideshow/preview_images_update.py
frm/ssh_all_farm.py
Modified: frm/blend_2_frames.py
===================================================================
--- frm/blend_2_frames.py 2010-05-05 14:20:48 UTC (rev 2915)
+++ frm/blend_2_frames.py 2010-05-05 14:42:30 UTC (rev 2916)
@@ -6,7 +6,10 @@
#frames/%s/%s_######" % (fname, fname)
def blend_2_frames(path):
- values = blend_render_info.read_blend_rend_chunk(path)
+ try:
+ values = blend_render_info.read_blend_rend_chunk(path)
+ except IOError:
+ return []
image_paths = []
fname = os.path.splitext(os.path.basename(path))[0]
Modified: frm/master_ui.py
===================================================================
--- frm/master_ui.py 2010-05-05 14:20:48 UTC (rev 2915)
+++ frm/master_ui.py 2010-05-05 14:42:30 UTC (rev 2916)
@@ -1,12 +1,14 @@
#!/shared/software/python/bin/python3.1
import cgi
+import datetime
import http.server
import os
import os.path
import pickle
import shutil
import subprocess
+import time
import urllib
from render_dirs import FARM_DIR
@@ -18,10 +20,8 @@
JOBS_FILE = os.path.join(FARM_DIR, "jobs.pkl")
SLAVES_FILE = os.path.join(FARM_DIR, "slaves.pkl")
REPO_PATH = "/media/data/svnroot/durian"
-
+HTTPD_IP = "192.168.1.14"
#HTTPD_IP = "127.0.0.1"
-HTTPD_IP = "192.168.1.14"
-
HTTPD_PORT = 8333
RESTART = False
GENERATE_AVI = False
@@ -121,6 +121,7 @@
job = Job(id, revision, quality)
JOBS.append(job)
+ JOBS.sort(key=lambda job: job.id)
Job.dump()
if MOVE_OLD_FRAMES:
@@ -177,6 +178,7 @@
return
SLAVES.append(Slave(id, ip))
+ SLAVES.sort(key=lambda slave: slave.id)
Slave.dump()
def slave_remove(id):
@@ -289,6 +291,7 @@
lines = f.readlines()[-maxlines:]
for line in lines:
self.wfile.write(bytes(line, encoding='utf8'))
+ lines.reverse()
f.close()
else:
f = open(path, 'rb')
@@ -308,6 +311,10 @@
return_file(os.path.join(FARM_DIR, 'slideshow/preview_big00001.png'), 'image/png')
elif self.path.startswith("/logs"):
return_file(os.path.join(FARM_DIR, self.path[1:]), 'text/plain', 1000)
+ elif self.path == '/graph.html':
+ return_file('frames/rrd/graph.html', 'text/html')
+ elif self.path.endswith('png'):
+ return_file('frames/rrd/%s' % self.path, 'image/png')
else:
# headers
self.send_response(http.client.OK)
@@ -316,7 +323,7 @@
# html begin
output("<html><head>\n")
- output("<meta http-equiv='refresh' content=5>\n")
+ output("<meta http-equiv='refresh' content=20>\n")
output("<title>Durian Farm! </title>\n")
output("<link rel='stylesheet' href='master_ui.css' type='text/css'>\n")
output("<script src='master_ui.js' type='text/javascript'></script>")
@@ -384,22 +391,34 @@
# progress
section("Overall Progress")
output(TOTAL_PROGRESS)
+ output("<br/><br/>")
+ section("Monitor")
+ output("<a href='graph.html'/>Node CPU/Memory Graphs.</a><br/>\n")
+
output("</div>\n")
# latest image
output("<div id='latest-render'>")
section("Latest Render")
output("<a href='preview_big.png' target='_blank'><img src='preview.png' alt='latest render image'></a>")
+ output("<br/><br/>")
output("</div>")
output("<hr/>\n")
# slaves
section("Slaves")
- table_begin("", "Name", "IP", "Status", "Logs", "")
+ table_begin("", "Name", "IP", "Status", "Since", "Logs", "")
for slave in SLAVES:
+ try:
+ busyfile = os.path.join(FARM_DIR, "./logs/%s.busy" % slave.ip)
+ secdiff = round(time.time() - os.stat(busyfile).st_mtime)
+ since = str(datetime.timedelta(0, secdiff))
+ except OSError:
+ since = ""
+
output("<tr>\n")
output("<td id='td-toggle'>")
checkbox('slave_set_enabled', slave.id, slave.enabled)
@@ -410,7 +429,10 @@
output("<td style='color:red'>" + slave.status + "</td>\n")
else:
output("<td>" + slave.status + "</td>\n")
- output("<td><a href='/logs/%s.log'>view</a></td>\n" % (slave.ip,))
+ output("<td>" + since + "</td>\n")
+ output("<td><a href='/logs/%s.log'>log</a></td>\n" % (slave.ip,))
+ #output("<a href='%s_cpu.png'>cpu</a>\n" % (slave.ip,))
+ #output("<a href='%s_mem.png'>mem</a></td>\n" % (slave.ip,))
output("<td id='td-toggle'>")
action('slave_remove(\\"%s\\")' % slave.id, "X", "Are you sure you want to remove this slave?")
output("</td>\n")
Modified: frm/new_blend_2_frames.py
===================================================================
--- frm/new_blend_2_frames.py 2010-05-05 14:20:48 UTC (rev 2915)
+++ frm/new_blend_2_frames.py 2010-05-05 14:42:30 UTC (rev 2916)
@@ -6,7 +6,10 @@
#frames/%s/%s_######" % (fname, fname)
def blend_2_frames(path):
- values = blend_render_info.read_blend_rend_chunk(path)
+ try:
+ values = blend_render_info.read_blend_rend_chunk(path)
+ except IOError:
+ return []
images = []
fname = os.path.splitext(os.path.basename(path))[0]
Modified: frm/new_blender_setup.py
===================================================================
--- frm/new_blender_setup.py 2010-05-05 14:20:48 UTC (rev 2915)
+++ frm/new_blender_setup.py 2010-05-05 14:42:30 UTC (rev 2916)
@@ -40,6 +40,8 @@
os.system("rm -rf %s" % tmpdir)
os.system("mkdir %s" % tmpdir)
+bpy.context.user_preferences.system.image_tile_memory_limit = 256
+
for scene in bpy.data.scenes:
rd = scene.render
@@ -48,7 +50,7 @@
rd.file_format = 'OPEN_EXR'
rd.exr_half = True
- rd.output_path = "/shared/software/durian_farm/frames/%s/%s_######" % (fname, fname)
+ rd.output_path = "/home/brecht/Desktop/frames/%s/%s_######" % (fname, fname)
rd.use_file_extension = True
rd.use_placeholder = False
@@ -76,6 +78,14 @@
# rd.alpha_mode = 'SKY'
rd.color_mode = 'RGB'
+ # performance/memory
+ rd.free_image_textures = True
+ rd.save_buffers = True
+ rd.full_sample = True
+ rd.threads_mode = 'AUTO'
+ rd.use_local_coords = False
+ if rd.parts_x < 32: rd.parts_x = 32
+ if rd.parts_y < 16: rd.parts_y = 16
if bpy.app.debug:
rd.resolution_percentage = 50
Modified: frm/new_master.py
===================================================================
--- frm/new_master.py 2010-05-05 14:20:48 UTC (rev 2915)
+++ frm/new_master.py 2010-05-05 14:42:30 UTC (rev 2916)
@@ -4,6 +4,7 @@
# * auto-generate avis after job is done, non-blocking
# * detect crashes so it does not get stuck on these frames
# * link durian directory to durian_farm_svn on nodes
+# * still using local svn to check for blend_2_frames
import os
import time
@@ -110,7 +111,7 @@
if image_done == 0:
job.status = "No Frames"
else:
- job.status = "Done"
+ job.status = "Done!"
elif not job.enabled:
job.status = "Disabled"
else:
Modified: frm/new_node_update_clear.sh
===================================================================
--- frm/new_node_update_clear.sh 2010-05-05 14:20:48 UTC (rev 2915)
+++ frm/new_node_update_clear.sh 2010-05-05 14:42:30 UTC (rev 2916)
@@ -9,8 +9,7 @@
rm $BUSY
pkill -P $(pgrep -f new_node_command.sh)
+killall -9 blender_farm
rm -rf /tmp/durian_farm
-rm -rf /tmp/*.exr
-rm -rf /tmp/*.blend
Modified: frm/new_node_update_movie.sh
===================================================================
--- frm/new_node_update_movie.sh 2010-05-05 14:20:48 UTC (rev 2915)
+++ frm/new_node_update_movie.sh 2010-05-05 14:42:30 UTC (rev 2916)
@@ -6,7 +6,7 @@
DIR="/media/data/durian_farm_svn"
-echo "Updating Movie Files" > $BUSY
+echo "Updating SVN Production Files" > $BUSY
if [ -d $DIR/pro ]; then
cd $DIR/pro
Modified: frm/slideshow/preview_images_update.py
===================================================================
--- frm/slideshow/preview_images_update.py 2010-05-05 14:20:48 UTC (rev 2915)
+++ frm/slideshow/preview_images_update.py 2010-05-05 14:42:30 UTC (rev 2916)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/shared/software/python/bin/python3.1
import os
import time
@@ -7,17 +7,16 @@
CONVERT = "/shared/software/durian_farm/slideshow/exr_to_png.sh %s %s %d > /dev/null 2> /dev/null"
PNG_FILE = "/shared/software/durian_farm/slideshow/preview"
NUM_IMAGES = 2
-NUM_SMALL_IMAGES = 2
SLEEP = 120.0
while 1:
- print "scanning for exr files"
+ print("scanning for exr files")
# find images to display
exr_files = []
for root, dirs, files in os.walk(DIR):
- print root
+ print(root)
for file in files:
if file.endswith('.exr'):
@@ -33,18 +32,18 @@
# convert images
for i in range(0, NUM_IMAGES):
- name, mtime = exr_files[i]
- print "converting", name
+ name, mtime = exr_files[0]
+ print("converting big", name)
command = CONVERT % (name, PNG_FILE + "_big" + str(i), 100)
os.system(command)
- for i in range(0, NUM_SMALL_IMAGES):
- name, mtime = exr_files[i]
- print "converting small", name
+ for i in range(0, NUM_IMAGES):
+ name, mtime = exr_files[0]
+ print("converting small", name)
command = CONVERT % (name, PNG_FILE + "_small" + str(i), 30)
os.system(command)
# sleep a while until the next up
- print "sleeping for", str(SLEEP/60.0), "minutes"
+ print("sleeping for", str(SLEEP/60.0), "minutes")
time.sleep(SLEEP)
Modified: frm/ssh_all_farm.py
===================================================================
--- frm/ssh_all_farm.py 2010-05-05 14:20:48 UTC (rev 2915)
+++ frm/ssh_all_farm.py 2010-05-05 14:42:30 UTC (rev 2916)
@@ -1,5 +1,12 @@
#!/usr/bin/python
import os
from ips import ips_all_farm as ips_all
-cmd = "cssh -l root " + " ".join(["%s:22" % ip for ip in ips_all])
+
+#
+ips_all = []
+for x in range(8):
@@ Diff output truncated at 10240 characters. @@
More information about the Durian-svn
mailing list