[Bf-extensions-cvs] SVN commit: /data/svn/bf-extensions [2803] trunk/py/scripts/addons/ netrender: netrender
Martin Poirier
theeth at yahoo.com
Fri Dec 23 01:35:35 CET 2011
Revision: 2803
http://projects.blender.org/scm/viewvc.php?view=rev&root=bf-extensions&revision=2803
Author: theeth
Date: 2011-12-23 00:35:24 +0000 (Fri, 23 Dec 2011)
Log Message:
-----------
netrender
- fix handling of pointcache dependencies on slave (correct repathing) and on client (bad frame dependency range)
- add Force Upload option on master to force dependencies upload and download (don't reuse local copies outside of slave cache)
- add links back to main page on job web page
- client limit dependencies to rendered frame range (it used to add all point cache frames)
Thanks to Philippe Van Hecke for raising the issue by email.
Modified Paths:
--------------
trunk/py/scripts/addons/netrender/client.py
trunk/py/scripts/addons/netrender/master.py
trunk/py/scripts/addons/netrender/master_html.py
trunk/py/scripts/addons/netrender/model.py
trunk/py/scripts/addons/netrender/repath.py
trunk/py/scripts/addons/netrender/slave.py
trunk/py/scripts/addons/netrender/ui.py
trunk/py/scripts/addons/netrender/utils.py
Modified: trunk/py/scripts/addons/netrender/client.py
===================================================================
--- trunk/py/scripts/addons/netrender/client.py 2011-12-22 16:06:28 UTC (rev 2802)
+++ trunk/py/scripts/addons/netrender/client.py 2011-12-23 00:35:24 UTC (rev 2803)
@@ -39,16 +39,14 @@
# fluid frames starts at 0, which explains the +1
# This is stupid
current_frame = int(match.groups()[1]) + 1
- job.addFile(path + fluid_file, current_frame, current_frame)
+ job.addFile(os.path.join(path, fluid_file), current_frame, current_frame)
def addPointCache(job, ob, point_cache, default_path):
if not point_cache.use_disk_cache:
return
- name = point_cache.name
- if name == "":
- name = "".join(["%02X" % ord(c) for c in ob.name])
+ name = cacheName(ob, point_cache)
cache_path = bpy.path.abspath(point_cache.filepath) if point_cache.use_external else default_path
@@ -70,7 +68,7 @@
if len(cache_files) == 1:
cache_frame, cache_file = cache_files[0]
- job.addFile(cache_path + cache_file, cache_frame, cache_frame)
+ job.addFile(os.path.join(cache_path, cache_file), cache_frame, cache_frame)
else:
for i in range(len(cache_files)):
current_item = cache_files[i]
@@ -79,18 +77,18 @@
current_frame, current_file = current_item
- if not next_item and not previous_item:
- job.addFile(cache_path + current_file, current_frame, current_frame)
+ if not next_item and not previous_item:
+ job.addFile(os.path.join(cache_path, current_file), current_frame, current_frame)
elif next_item and not previous_item:
next_frame = next_item[0]
- job.addFile(cache_path + current_file, current_frame, next_frame - 1)
+ job.addFile(os.path.join(cache_path, current_file), current_frame, next_frame)
elif not next_item and previous_item:
previous_frame = previous_item[0]
- job.addFile(cache_path + current_file, previous_frame + 1, current_frame)
+ job.addFile(os.path.join(cache_path, current_file), previous_frame, current_frame)
else:
next_frame = next_item[0]
previous_frame = previous_item[0]
- job.addFile(cache_path + current_file, previous_frame + 1, next_frame - 1)
+ job.addFile(os.path.join(cache_path, current_file), previous_frame, next_frame)
def fillCommonJobSettings(job, job_name, netsettings):
job.name = job_name
@@ -219,8 +217,7 @@
###########################
# FLUID + POINT CACHE
###########################
- root, ext = os.path.splitext(name)
- default_path = path + os.sep + "blendcache_" + root + os.sep # need an API call for that
+ default_path = cachePath(filename)
for object in bpy.data.objects:
for modifier in object.modifiers:
@@ -294,7 +291,13 @@
address = "" if netsettings.server_address == "[default]" else netsettings.server_address
- master.runMaster((address, netsettings.server_port), netsettings.use_master_broadcast, netsettings.use_master_clear, bpy.path.abspath(netsettings.path), self.update_stats, self.test_break)
+ master.runMaster(address = (address, netsettings.server_port),
+ broadcast = netsettings.use_master_broadcast,
+ clear = netsettings.use_master_clear,
+ force = netsettings.use_master_force_upload,
+ path = bpy.path.abspath(netsettings.path),
+ update_stats = self.update_stats,
+ test_break = self.test_break)
def render_slave(self, scene):
Modified: trunk/py/scripts/addons/netrender/master.py
===================================================================
--- trunk/py/scripts/addons/netrender/master.py 2011-12-22 16:06:28 UTC (rev 2802)
+++ trunk/py/scripts/addons/netrender/master.py 2011-12-23 00:35:24 UTC (rev 2803)
@@ -34,15 +34,25 @@
super().__init__(filepath, index, start, end, signature)
self.found = False
- def test(self):
+ def updateStatus(self):
self.found = os.path.exists(self.filepath)
+
if self.found and self.signature != None:
found_signature = hashFile(self.filepath)
self.found = self.signature == found_signature
+ if not self.found:
+ print("Signature mismatch", self.signature, found_signature)
return self.found
+ def test(self):
+ # don't check when forcing upload and only until found
+ if not self.force and not self.found:
+ self.updateStatus()
+
+ return self.found
+
class MRenderSlave(netrender.model.RenderSlave):
def __init__(self, name, address, stats):
super().__init__()
@@ -86,6 +96,10 @@
self.last_update = 0
self.save_path = ""
self.files = [MRenderFile(rfile.filepath, rfile.index, rfile.start, rfile.end, rfile.signature) for rfile in job_info.files]
+
+ def setForceUpload(self, force):
+ for rfile in self.files:
+ rfile.force = force
def initInfo(self):
if not self.resolution:
@@ -514,6 +528,8 @@
job_id = self.server.nextJobID()
job = MRenderJob(job_id, job_info)
+
+ job.setForceUpload(self.server.force)
for frame in job_info.frames:
frame = job.addFrame(frame.number, frame.command)
@@ -701,9 +717,8 @@
match = file_pattern.match(self.path)
if match:
- self.server.stats("", "Receiving job")
+ self.server.stats("", "Receiving job file")
- length = int(self.headers['content-length'])
job_id = match.groups()[0]
file_index = int(match.groups()[1])
@@ -719,7 +734,7 @@
main_path, main_name = os.path.split(main_file)
if file_index > 0:
- file_path = prefixPath(job.save_path, render_file.filepath, main_path)
+ file_path = prefixPath(job.save_path, render_file.filepath, main_path, force = True)
else:
file_path = os.path.join(job.save_path, main_name)
@@ -728,12 +743,16 @@
self.write_file(file_path)
render_file.filepath = file_path # set the new path
-
- if job.testStart():
+ found = render_file.updateStatus() # make sure we have the right file
+
+ if not found: # checksum mismatch
+ self.server.stats("", "File upload but checksum mismatch, this shouldn't happen")
+ self.send_head(http.client.CONFLICT)
+ elif job.testStart(): # started correctly
self.server.stats("", "File upload, starting job")
self.send_head(content = None)
else:
- self.server.stats("", "File upload, file missings")
+ self.server.stats("", "File upload, dependency files still missing")
self.send_head(http.client.ACCEPTED)
else: # invalid file
print("file not found", job_id, file_index)
@@ -851,12 +870,13 @@
self.send_head(http.client.NO_CONTENT)
class RenderMasterServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
- def __init__(self, address, handler_class, path, subdir=True):
+ def __init__(self, address, handler_class, path, force=False, subdir=True):
self.jobs = []
self.jobs_map = {}
self.slaves = []
self.slaves_map = {}
self.job_id = 0
+ self.force = force
if subdir:
self.path = os.path.join(path, "master_" + str(os.getpid()))
@@ -1012,7 +1032,7 @@
def clearMaster(path):
shutil.rmtree(path)
-def createMaster(address, clear, path):
+def createMaster(address, clear, force, path):
filepath = os.path.join(path, "blender_master.data")
if not clear and os.path.exists(filepath):
@@ -1020,12 +1040,12 @@
with open(filepath, 'rb') as f:
path, jobs, slaves = pickle.load(f)
- httpd = RenderMasterServer(address, RenderHandler, path, subdir=False)
+ httpd = RenderMasterServer(address, RenderHandler, path, force=force, subdir=False)
httpd.restore(jobs, slaves)
return httpd
- return RenderMasterServer(address, RenderHandler, path)
+ return RenderMasterServer(address, RenderHandler, path, force=force)
def saveMaster(path, httpd):
filepath = os.path.join(path, "blender_master.data")
@@ -1033,8 +1053,8 @@
with open(filepath, 'wb') as f:
pickle.dump((httpd.path, httpd.jobs, httpd.slaves), f, pickle.HIGHEST_PROTOCOL)
-def runMaster(address, broadcast, clear, path, update_stats, test_break):
- httpd = createMaster(address, clear, path)
+def runMaster(address, broadcast, clear, force, path, update_stats, test_break):
+ httpd = createMaster(address, clear, force, path)
httpd.timeout = 1
httpd.stats = update_stats
Modified: trunk/py/scripts/addons/netrender/master_html.py
===================================================================
--- trunk/py/scripts/addons/netrender/master_html.py 2011-12-22 16:06:28 UTC (rev 2802)
+++ trunk/py/scripts/addons/netrender/master_html.py 2011-12-23 00:35:24 UTC (rev 2803)
@@ -215,6 +215,8 @@
job_id = handler.path[9:]
head("NetRender")
+
+ output(link("Back to Main Page", "/html"))
job = handler.server.getJobID(job_id)
@@ -311,5 +313,7 @@
else:
output("no such job")
+ output(link("Back to Main Page", "/html"))
+
output("</body></html>")
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-extensions-cvs
mailing list