[Durian-svn] [2622] durian render farm now in svn
campbell
institute at blender.org
Thu Apr 29 15:01:55 CEST 2010
Revision: 2622
https://blenderinstitute.dyndns.org/durian-svn/?do=log&project=durian&path=/&rev=2622
Author: campbell
Date: 2010-04-29 15:01:55 +0200 (Thu, 29 Apr 2010)
Log Message:
-----------
durian render farm now in svn
Added Paths:
-----------
frm/
frm/blend_2_frames.py
frm/blend_render_info.py
frm/blender_exr_to_avi.py
frm/blender_setup.py
frm/ips.py
frm/master.py
frm/master_avi_gen.py
frm/node_update_blender.sh
frm/node_update_clear.sh
frm/node_update_frames.sh
frm/node_update_movie.sh
frm/render_dirs.py
frm/ssh_all.py
frm/ssh_all_farm.py
frm/svn_check_renders.py
Added: frm/blend_2_frames.py
===================================================================
--- frm/blend_2_frames.py (rev 0)
+++ frm/blend_2_frames.py 2010-04-29 13:01:55 UTC (rev 2622)
@@ -0,0 +1,36 @@
+import blend_render_info
+import os
+
+from render_dirs import FARM_DIR
+
+#frames/%s/%s_######" % (fname, fname)
+
+def blend_2_frames(path):
+ values = blend_render_info.read_blend_rend_chunk(path)
+
+ image_paths = []
+ fname = os.path.splitext(os.path.basename(path))[0]
+
+ for start, end, scene in values:
+ format_string = os.path.join(FARM_DIR, "frames", fname, fname + "_%.6d.exr")
+ frame = start
+ while frame <= end:
+ image_paths.append(format_string % frame)
+ frame += 1
+
+ return image_paths
+
+
+def main():
+ import sys
+ for arg in sys.argv[1:]:
+ if arg.lower().endswith('.blend'):
+ for path in blend_2_frames(arg):
+ print(path)
+
+
+if __name__ == '__main__':
+ main()
+
+#for path in blend_2_frames("/shared/software/durian_farm/pro/comps/03.1_alley/03.1b.blend"):
+# print path
Property changes on: frm/blend_2_frames.py
___________________________________________________________________
Added: svn:executable
+ *
Added: frm/blend_render_info.py
===================================================================
--- frm/blend_render_info.py (rev 0)
+++ frm/blend_render_info.py 2010-04-29 13:01:55 UTC (rev 2622)
@@ -0,0 +1,100 @@
+#!/usr/bin/python
+
+# ##### BEGIN GPL LICENSE BLOCK #####
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ##### END GPL LICENSE BLOCK #####
+
+# <pep8 compliant>
+
+# In Blender, selecting scenes in the databrowser (shift+f4) will tag for rendering.
+
+# This struct wont change according to ton.
+# Note that the size differs on 32/64bit
+'''
+typedef struct BHead {
+ int code, len;
+ void *old;
+ int SDNAnr, nr;
+} BHead;
+'''
+
+def read_blend_rend_chunk(path):
+
+ import struct
+
+ file = open(path, 'rb')
+
+ head = file.read(7)
+
+ if head[0:2] == b'\x1f\x8b': # gzip magic
+ import gzip
+ file.close()
+ file = gzip.open(path, 'rb')
+ head = file.read(7)
+
+ if head != b'BLENDER':
+ print("not a blend file:", path)
+ file.close()
+ return []
+
+ is_64_bit = (file.read(1) == b'-')
+
+ # true for PPC, false for X86
+ is_big_endian = (file.read(1) == b'V')
+
+ # Now read the bhead chunk!!!
+ file.read(3) # skip the version
+
+ scenes = []
+
+ sizeof_bhead = 24 if is_64_bit else 20
+
+ while file.read(4) == b'REND':
+ sizeof_bhead_left = sizeof_bhead - 4
+
+ rend_length = struct.unpack('>i' if is_big_endian else '<i', file.read(4))[0]
+ sizeof_bhead_left -= 4
+
+ # We dont care about the rest of the bhead struct
+ file.read(sizeof_bhead_left)
+
+ # Now we want the scene name, start and end frame. this is 32bites long
+ start_frame, end_frame = struct.unpack('>2i' if is_big_endian else '<2i', file.read(8))
+
+ scene_name = file.read(24)
+
+ scene_name = scene_name[:scene_name.index(b'\0')]
+
+ try:
+ scene_name = str(scene_name, 'utf8')
+ except TypeError:
+ pass
+
+ scenes.append((start_frame, end_frame, scene_name))
+
+ return scenes
+
+
+def main():
+ import sys
+ for arg in sys.argv[1:]:
+ if arg.lower().endswith('.blend'):
+ for value in read_blend_rend_chunk(arg):
+ print("%d %d %s" % value)
+
+if __name__ == '__main__':
+ main()
Property changes on: frm/blend_render_info.py
___________________________________________________________________
Added: svn:executable
+ *
Added: frm/blender_exr_to_avi.py
===================================================================
--- frm/blender_exr_to_avi.py (rev 0)
+++ frm/blender_exr_to_avi.py 2010-04-29 13:01:55 UTC (rev 2622)
@@ -0,0 +1,24 @@
+import sys
+image_dir = sys.argv[-1]
+# image_dir = "/shared/software/durian_farm/frames/03.4f/"
+
+if not image_dir.endswith("/"):
+ image_dir += "/"
+
+import os
+files = [{"name": f} for f in sorted(os.listdir(image_dir)) if f.endswith(".exr")]
+bpy.ops.sequencer.image_strip_add(path=image_dir, filename="test.blend", directory=image_dir, filter_blender=False, filter_image=True, filter_movie=False, name="", frame_start=1, channel=1, replace_sel=True, files=files)
+
+# render
+bpy.context.scene.frame_start = 1
+bpy.context.scene.frame_end = len(files)
+bpy.context.scene.render.use_sequencer = True
+bpy.context.scene.render.resolution_percentage = 50
+bpy.context.scene.render.resolution_x = 2048
+bpy.context.scene.render.resolution_y = 872
+bpy.context.scene.render.file_format = 'AVI_JPEG'
+bpy.context.scene.render.file_quality = 95
+bpy.context.scene.render.output_path = image_dir
+
+
+bpy.ops.render.render(animation=True)
\ No newline at end of file
Property changes on: frm/blender_exr_to_avi.py
___________________________________________________________________
Added: svn:executable
+ *
Added: frm/blender_setup.py
===================================================================
--- frm/blender_setup.py (rev 0)
+++ frm/blender_setup.py 2010-04-29 13:01:55 UTC (rev 2622)
@@ -0,0 +1,103 @@
+# runs on the nodes inside blender
+
+import bpy
+import os
+
+def get_ip():
+
+ import socket
+ import fcntl
+ import struct
+
+ def get_ip_address(ifname):
+ s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ return socket.inet_ntoa(fcntl.ioctl(
+ s.fileno(),
+ 0x8915, # SIOCGIFADDR
+ struct.pack('256s', ifname[:15])
+ )[20:24])
+
+ for iface in "eth0", "eth1", "eth2", "eth3":
+ try:
+ ip = get_ip_address(iface)
+ break
+ except:
+ ip = None
+
+ return ip
+
+ip = get_ip()
+fpath = bpy.data.filename
+fname = os.path.splitext(os.path.basename(fpath))[0]
+scene_current = bpy.context.scene
+
+print("loaded:", fpath, scene_current)
+
+# not working
+bpy.context.user_preferences.filepaths.temporary_directory = "/tmp/durian_farm"
+os.system("rm -rf /tmp/durian_farm")
+os.system("mkdir /tmp/durian_farm")
+
+for scene in bpy.data.scenes:
+ rd = scene.render
+
+ # file stuff...
+ override = False # (rd.file_format != 'PNG')
+
+ rd.file_format = 'OPEN_EXR'
+ rd.exr_half = True
+ rd.output_path = "/shared/software/durian_farm/frames/%s/%s_######" % (fname, fname)
+ rd.use_file_extension = True
+
+ rd.use_placeholder = True
+ rd.use_overwrite = False
+
+
+ rd.render_stamp = False
+ rd.stamp_note = True
+ rd.stamp_render_time = True
+ rd.stamp_note_text = "rev:%s, %s" % (bpy.app.build_revision, ip)
+ rd.stamp_font_size = 18
+ rd.stamp_foreground = 1.0, 1.0, 1.0, 1.0
+ rd.stamp_background = 0.0, 0.0, 0.0, 0.75
+
+ if override:
+ continue
+
+ # render settings...
+ rd.resolution_percentage = 100
+ rd.resolution_x = 2048
+ rd.resolution_y = 872
+
+ rd.use_border = False
+# rd.color_management = False
+# rd.alpha_mode = 'SKY'
+ rd.color_mode = 'RGB'
+
+
+ if bpy.app.debug:
+ rd.resolution_percentage = 50
+ # rd.simplify_child_particles = 0.0
+ #rd.simplify_shadow_samples = 0
+ #rd.simplify_subdivision = 0
+ rd.simplify_triangulate = True
+ rd.use_simplify = True
+
+ #rd.use_textures = False
+ #rd.use_raytracing = False
+ #rd.use_sss = False
+ #rd.use_shadows = False
+
+ # rd.file_format = 'PNG'
+
+ # special ben check
+ '''
+ if ip.endswith(".12"):
+ if "nautilus" in os.popen("ps -A").read():
+ rd.threads_mode = 'FIXED'
+ rd.threads = 12
+ '''
+
+
+print("rendering rendering %d" % (scene_current.frame_end - scene_current.frame_start))
+bpy.ops.render.render(animation=True)
Property changes on: frm/blender_setup.py
___________________________________________________________________
Added: svn:executable
+ *
Added: frm/ips.py
===================================================================
--- frm/ips.py (rev 0)
+++ frm/ips.py 2010-04-29 13:01:55 UTC (rev 2622)
@@ -0,0 +1,24 @@
+ips_all = []
+
+'''
+ips_all += ["192.168.1.11"] # brecht's/amd
+ips_all += ["192.168.1.12"] # ben
+#ips_all += ["192.168.1.13"] # brecht
+ips_all += ["192.168.1.15"] # soenke
+#ips_all += ["192.168.1.19"] # weekly
+ips_all += ["192.168.1.20"] # ideasman
+#ips_all += ["192.168.1.21"] # pablo
+ips_all += ["192.168.1.22"] # colin
+ips_all += ["192.168.1.23"] # buourne, 2gig!
+ips_all += ["192.168.1.24"] # lee
+ips_all += ["192.168.1.25"] # angela
+ips_all += ["192.168.1.26"] # i-7, closest to server room
+ips_all += ["192.168.1.27"] # nathan
+'''
+
+ips_all += ["192.168.1.201"]
+ips_all += ["192.168.1.202"]
+ips_all += ["192.168.1.203"]
+ips_all += ["192.168.1.204"]
+
+ips_all_farm = ips_all[:]
\ No newline at end of file
Added: frm/master.py
===================================================================
--- frm/master.py (rev 0)
+++ frm/master.py 2010-04-29 13:01:55 UTC (rev 2622)
@@ -0,0 +1,325 @@
+#!/shared/software/python/bin/python3.1
+'''
+TODO.
+- progress %
+- check if systems are logged in and add/remove systems on the fly
+- remove zero length images on finish
+- make exr preview avis
+
+'''
+
+
+import os
+import time
+
+os.umask(777)
+
+from render_dirs import FARM_DIR
+
+# cat /shared/software/durian_farm/id_dsa.pub > /home/guest/.ssh/authorized_keys
+
+DEBUG = True
+
+files = []
+
+from ips import ips_all
+
+ips_all = set(ips_all)
+
+def log_file(ip, ext="log"):
+ return "%s/logs/%s.%s" % (FARM_DIR, ip, ext)
+
@@ Diff output truncated at 10240 characters. @@
More information about the Durian-svn
mailing list