[Bf-blender-cvs] SVN commit: /data/svn/bf-blender [23357] trunk/blender/release/io/netrender : netrender: first draft for job balancer + some minor fixes
Martin Poirier
theeth at yahoo.com
Sun Sep 20 00:11:26 CEST 2009
Revision: 23357
http://projects.blender.org/plugins/scmsvn/viewcvs.php?view=rev&root=bf-blender&revision=23357
Author: theeth
Date: 2009-09-20 00:11:26 +0200 (Sun, 20 Sep 2009)
Log Message:
-----------
netrender: first draft for job balancer + some minor fixes
Modified Paths:
--------------
trunk/blender/release/io/netrender/__init__.py
trunk/blender/release/io/netrender/client.py
trunk/blender/release/io/netrender/master.py
trunk/blender/release/io/netrender/model.py
trunk/blender/release/io/netrender/operators.py
trunk/blender/release/io/netrender/slave.py
trunk/blender/release/io/netrender/utils.py
Added Paths:
-----------
trunk/blender/release/io/netrender/balancing.py
Modified: trunk/blender/release/io/netrender/__init__.py
===================================================================
--- trunk/blender/release/io/netrender/__init__.py 2009-09-19 22:02:15 UTC (rev 23356)
+++ trunk/blender/release/io/netrender/__init__.py 2009-09-19 22:11:26 UTC (rev 23357)
@@ -6,6 +6,7 @@
import slave
import master
import utils
+import balancing
import ui
# store temp data in bpy module
Added: trunk/blender/release/io/netrender/balancing.py
===================================================================
--- trunk/blender/release/io/netrender/balancing.py (rev 0)
+++ trunk/blender/release/io/netrender/balancing.py 2009-09-19 22:11:26 UTC (rev 23357)
@@ -0,0 +1,75 @@
+import time
+
+from netrender.utils import *
+import netrender.model
+
+class RatingRule:
+ def rate(self, job):
+ return 0
+
+class ExclusionRule:
+ def test(self, job):
+ return False
+
+class PriorityRule:
+ def test(self, job):
+ return False
+
+class Balancer:
+ def __init__(self):
+ self.rules = []
+ self.priorities = []
+ self.exceptions = []
+
+ def addRule(self, rule):
+ self.rules.append(rule)
+
+ def addPriority(self, priority):
+ self.priorities.append(priority)
+
+ def addException(self, exception):
+ self.exceptions.append(exception)
+
+ def applyRules(self, job):
+ return sum((rule.rate(job) for rule in self.rules))
+
+ def applyPriorities(self, job):
+ for priority in self.priorities:
+ if priority.test(job):
+ return True # priorities are first
+
+ return False
+
+ def applyExceptions(self, job):
+ for exception in self.exceptions:
+ if exception.test(job):
+ return True # exceptions are last
+
+ return False
+
+ def sortKey(self, job):
+ return (1 if self.applyExceptions(job) else 0, # exceptions after
+ 0 if self.applyPriorities(job) else 1, # priorities first
+ self.applyRules(job))
+
+ def balance(self, jobs):
+ if jobs:
+ jobs.sort(key=self.sortKey)
+ return jobs[0]
+ else:
+ return None
+
+# ==========================
+
+
+class RatingCredit(RatingRule):
+ def rate(self, job):
+ return -job.credits # more credit is better (sort at first in list)
+
+class NewJobPriority(PriorityRule):
+ def test(self, job):
+ return job.countFrames(status = DISPATCHED) == 0
+
+class ExcludeQueuedEmptyJob(ExclusionRule):
+ def test(self, job):
+ return job.status != JOB_QUEUED or job.countFrames(status = QUEUED) == 0
Property changes on: trunk/blender/release/io/netrender/balancing.py
___________________________________________________________________
Name: svn:keywords
+ Author Date Id Revision
Name: svn:eol-style
+ native
Modified: trunk/blender/release/io/netrender/client.py
===================================================================
--- trunk/blender/release/io/netrender/client.py 2009-09-19 22:02:15 UTC (rev 23356)
+++ trunk/blender/release/io/netrender/client.py 2009-09-19 22:11:26 UTC (rev 23357)
@@ -103,7 +103,7 @@
job.priority = netsettings.priority
# try to send path first
- conn.request("POST", "job", repr(job.serialize()))
+ conn.request("POST", "/job", repr(job.serialize()))
response = conn.getresponse()
job_id = response.getheader("job-id")
@@ -112,7 +112,7 @@
if response.status == http.client.ACCEPTED:
for filepath, start, end in job.files:
f = open(filepath, "rb")
- conn.request("PUT", "file", f, headers={"job-id": job_id, "job-file": filepath})
+ conn.request("PUT", "/file", f, headers={"job-id": job_id, "job-file": filepath})
f.close()
response = conn.getresponse()
@@ -121,7 +121,7 @@
return job_id
def requestResult(conn, job_id, frame):
- conn.request("GET", "render", headers={"job-id": job_id, "job-frame":str(frame)})
+ conn.request("GET", "/render", headers={"job-id": job_id, "job-frame":str(frame)})
@rnaType
class NetworkRenderEngine(bpy.types.RenderEngine):
@@ -174,7 +174,6 @@
requestResult(conn, job_id, scene.current_frame)
while response.status == http.client.ACCEPTED and not self.test_break():
- print("waiting")
time.sleep(1)
requestResult(conn, job_id, scene.current_frame)
response = conn.getresponse()
Modified: trunk/blender/release/io/netrender/master.py
===================================================================
--- trunk/blender/release/io/netrender/master.py 2009-09-19 22:02:15 UTC (rev 23356)
+++ trunk/blender/release/io/netrender/master.py 2009-09-19 22:11:26 UTC (rev 23357)
@@ -4,11 +4,8 @@
from netrender.utils import *
import netrender.model
+import netrender.balancing
-JOB_WAITING = 0 # before all data has been entered
-JOB_PAUSED = 1 # paused by user
-JOB_QUEUED = 2 # ready to be dispatched
-
class MRenderFile:
def __init__(self, filepath, start, end):
self.filepath = filepath
@@ -38,10 +35,6 @@
def seen(self):
self.last_seen = time.time()
-# sorting key for jobs
-def groupKey(job):
- return (job.status, job.framesLeft() > 0, job.priority, job.credits)
-
class MRenderJob(netrender.model.RenderJob):
def __init__(self, job_id, name, files, chunks = 1, priority = 1, credits = 100.0, blacklist = []):
super().__init__()
@@ -95,15 +88,7 @@
frame = MRenderFrame(frame_number)
self.frames.append(frame)
return frame
-
- def framesLeft(self):
- total = 0
- for j in self.frames:
- if j.status == QUEUED:
- total += 1
- return total
-
def reset(self, all):
for f in self.frames:
f.reset(all)
@@ -153,7 +138,7 @@
def do_HEAD(self):
print(self.path)
- if self.path == "status":
+ if self.path == "/status":
job_id = self.headers.get('job-id', "")
job_frame = int(self.headers.get('job-frame', -1))
@@ -185,12 +170,12 @@
def do_GET(self):
print(self.path)
- if self.path == "version":
+ if self.path == "/version":
self.send_head()
self.server.stats("", "New client connection")
self.wfile.write(VERSION)
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
- elif self.path == "render":
+ elif self.path == "/render":
job_id = self.headers['job-id']
job_frame = int(self.headers['job-frame'])
print("render:", job_id, job_frame)
@@ -221,7 +206,7 @@
# no such job id
self.send_head(http.client.NO_CONTENT)
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
- elif self.path == "log":
+ elif self.path == "/log":
job_id = self.headers['job-id']
job_frame = int(self.headers['job-frame'])
print("log:", job_id, job_frame)
@@ -250,7 +235,7 @@
# no such job id
self.send_head(http.client.NO_CONTENT)
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
- elif self.path == "status":
+ elif self.path == "/status":
job_id = self.headers.get('job-id', "")
job_frame = int(self.headers.get('job-frame', -1))
@@ -284,7 +269,7 @@
self.wfile.write(bytes(repr(message), encoding='utf8'))
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
- elif self.path == "job":
+ elif self.path == "/job":
self.server.update()
slave_id = self.headers['slave-id']
@@ -315,7 +300,7 @@
else: # invalid slave id
self.send_head(http.client.NO_CONTENT)
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
- elif self.path == "file":
+ elif self.path == "/file":
slave_id = self.headers['slave-id']
slave = self.server.updateSlave(slave_id)
@@ -348,7 +333,7 @@
else: # invalid slave id
self.send_head(http.client.NO_CONTENT)
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
- elif self.path == "slave":
+ elif self.path == "/slave":
message = []
for slave in self.server.slaves:
@@ -368,7 +353,7 @@
print(self.path)
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
- if self.path == "job":
+ if self.path == "/job":
print("posting job info")
self.server.stats("", "Receiving job")
@@ -394,7 +379,7 @@
else:
self.send_head(http.client.ACCEPTED, headers=headers)
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
- elif self.path == "cancel":
+ elif self.path == "/cancel":
job_id = self.headers.get('job-id', "")
if job_id:
print("cancel:", job_id, "\n")
@@ -404,7 +389,7 @@
self.send_head()
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
- elif self.path == "reset":
+ elif self.path == "/reset":
job_id = self.headers.get('job-id', "")
job_frame = int(self.headers.get('job-frame', "-1"))
all = bool(self.headers.get('reset-all', "False"))
@@ -421,7 +406,7 @@
else: # job not found
self.send_head(http.client.NO_CONTENT)
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
- elif self.path == "slave":
+ elif self.path == "/slave":
length = int(self.headers['content-length'])
job_frame_string = self.headers['job-frame']
@@ -431,7 +416,7 @@
self.send_head(headers = {"slave-id": slave_id})
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
- elif self.path == "log":
+ elif self.path == "/log":
slave_id = self.headers['slave-id']
slave = self.server.updateSlave(slave_id)
@@ -460,7 +445,7 @@
print(self.path)
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
- if self.path == "file":
+ if self.path == "/file":
print("writing blend file")
self.server.stats("", "Receiving job")
@@ -504,7 +489,7 @@
else: # job not found
self.send_head(http.client.NO_CONTENT)
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
- elif self.path == "render":
+ elif self.path == "/render":
print("writing result file")
self.server.stats("", "Receiving render result")
@@ -547,7 +532,7 @@
else: # invalid slave id
self.send_head(http.client.NO_CONTENT)
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
- elif self.path == "log":
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list