[Bf-extensions-cvs] SVN commit: /data/svn/bf-extensions [3675] trunk/py/scripts/tools/bi_farm: Renderfarm: show online status of slaves, also check for whether disk is full
Sergey Sharybin
sergey.vfx at gmail.com
Fri Aug 10 13:36:56 CEST 2012
Revision: 3675
http://projects.blender.org/scm/viewvc.php?view=rev&root=bf-extensions&revision=3675
Author: nazgul
Date: 2012-08-10 11:36:56 +0000 (Fri, 10 Aug 2012)
Log Message:
-----------
Renderfarm: show online status of slaves, also check for whether disk is full
Modified Paths:
--------------
trunk/py/scripts/tools/bi_farm/master_ui.py
trunk/py/scripts/tools/bi_farm/new_master.py
Added Paths:
-----------
trunk/py/scripts/tools/bi_farm/new_node_check_space.sh
Modified: trunk/py/scripts/tools/bi_farm/master_ui.py
===================================================================
--- trunk/py/scripts/tools/bi_farm/master_ui.py 2012-08-10 01:20:39 UTC (rev 3674)
+++ trunk/py/scripts/tools/bi_farm/master_ui.py 2012-08-10 11:36:56 UTC (rev 3675)
@@ -44,8 +44,8 @@
JOBS_FILE = os.path.join(FARM_DIR, "jobs.pkl")
SLAVES_FILE = os.path.join(FARM_DIR, "slaves.pkl")
REPO_PATH = "/extra/ltstor/svnroot/mango"
-HTTPD_IP = "192.168.3.14"
-#HTTPD_IP = "127.0.0.1"
+#HTTPD_IP = "192.168.3.14"
+HTTPD_IP = "127.0.0.1"
HTTPD_PORT = 8333
RESTART = False
GENERATE_AVI = False
@@ -270,6 +270,9 @@
self.is_cuda = is_cuda
self.status = "Disabled"
self.enabled = False
+ self.online = False
+ self.disk_space_ok = False
+ self.last_space_check = 0
@staticmethod
def find(id):
@@ -294,6 +297,12 @@
for slave in SLAVES:
if not hasattr(slave, "is_cuda"):
slave.is_cuda = False
+ if not hasattr(slave, "last_space_check"):
+ slave.last_space_check = 0
+ if not hasattr(slave, "online"):
+ slave.online = False
+ if not hasattr(slave, "disk_space_ok"):
+ slave.disk_space_ok = True
f.close()
except IOError:
@@ -619,7 +628,7 @@
# slaves
section("Slaves")
- table_begin("", "Name", "IP", "Cuda", "Status", "Since", "Logs", "")
+ table_begin("", "Name", "IP", "Alive", "Cuda", "Status", "Since", "Logs", "")
for slave in SLAVES:
try:
@@ -635,7 +644,14 @@
output("</td>\n")
output("<td>" + slave.id + "</td>\n")
output("<td>" + slave.ip + "</td>\n")
- output("<td>" + ("☑" if slave.is_cuda else "☐") + "</td>\n")
+ if slave.online:
+ if slave.disk_space_ok:
+ output("<td width=96 id=\"status-done\">Online</td>\n")
+ else:
+ output("<td width=96 id=\"status-inprogress\">Disk is Full</td>\n")
+ else:
+ output("<td width=96 id=\"status-disabled\">Offline</td>\n")
+ output("<td> " + ("☑" if slave.is_cuda else "☐") + "</td>\n")
if slave.status == 'Disabled':
output("<td style='color:red'>" + slave.status + "</td>\n")
else:
Modified: trunk/py/scripts/tools/bi_farm/new_master.py
===================================================================
--- trunk/py/scripts/tools/bi_farm/new_master.py 2012-08-10 01:20:39 UTC (rev 3674)
+++ trunk/py/scripts/tools/bi_farm/new_master.py 2012-08-10 11:36:56 UTC (rev 3675)
@@ -1,4 +1,4 @@
-#!/shared/software/python/bin/python3.2
+#!/opt/lib/python3.2/bin/python3.2
# ##### BEGIN GPL LICENSE BLOCK #####
#
@@ -32,6 +32,8 @@
import master_ui
import random
import new_blend_2_frames
+import socket
+import subprocess
from render_dirs import FARM_DIR
from render_dirs import REND_DIR
from render_dirs import GEN_AVI_BUSY
@@ -51,16 +53,24 @@
return "%s/logs/%s.%s" % (FARM_DIR, ip, ext)
-def remote_command(ip, cmd, user="guest", passwd="guest", log=True):
- logpath = log_file(ip)
+def get_user_for_ip(ip, user="guest"):
# node exception
- ip_groubs = ip.split(".")
- if int(ip_groubs[-2]) >= 4 and int(ip_groubs[-1]) >= 200:
+ ip_groups = ip.split(".")
+ if int(ip_groups[-2]) >= 4 and int(ip_groups[-1]) >= 200:
user = "root"
elif ip == "192.168.4.14":
user = "user"
+ return user
+
+
+def remote_command(ip, cmd, user="guest", passwd="guest", log=True):
+ logpath = log_file(ip)
+
+ # some ip's are using different user
+ user = get_user_for_ip(ip, user)
+
# add redirection
if log:
cmd += " >> %s 2>&1" % logpath
@@ -396,6 +406,34 @@
os.system("rm %s" % busypath)
+def slave_check_online(slave):
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.settimeout(0.1)
+ try:
+ s.connect((slave.ip, 22))
+ s.close()
+ return True
+ except:
+ return False
+
+
+def slave_check_disk_space(slave):
+ now = time.time()
+ if now - slave.last_space_check < 30 * 60:
+ # check disk usage once in half of hour only
+ return slave.disk_space_ok
+
+ user = get_user_for_ip(slave.ip)
+
+ script = os.path.join(FARM_DIR, "new_node_check_space.sh")
+ ip = user + "@" + slave.ip
+
+ print("Check disk usage for %s" % slave.ip)
+ with subprocess.Popen([script, ip], stdout=subprocess.PIPE) as proc:
+ stat = proc.stdout.read().decode().strip()
+ slave.last_space_check = now
+ return stat == "OK"
+
def stage_in():
'''
@@ -433,6 +471,11 @@
changed = False
+ for slave in master_ui.SLAVES:
+ slave.online = slave_check_online(slave)
+ if slave.online:
+ slave.disk_space_ok = slave_check_disk_space(slave)
+
# add/remove available slaves
slaves = available_slaves()
@@ -482,7 +525,7 @@
changed = True
# find a new job
- if slave.enabled and slave_func_nr == 0:
+ if slave.online and slave.enabled and slave_func_nr == 0:
do_finals = slave.ip.startswith("192.168.1.2")
if slave.ip.startswith("192.168.4.7"):
if int(slave.ip.split(".")[-1]) in (75, 76):
Added: trunk/py/scripts/tools/bi_farm/new_node_check_space.sh
===================================================================
--- trunk/py/scripts/tools/bi_farm/new_node_check_space.sh (rev 0)
+++ trunk/py/scripts/tools/bi_farm/new_node_check_space.sh 2012-08-10 11:36:56 UTC (rev 3675)
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+IP=$1
+
+disk_free=`ssh $IP df /media/data/mango_farm_svn/ | tail -n 1 | sed -r 's/\s+/ /g' | cut -d \ -f 4`
+
+# 5GB should be enough for farm
+if [[ "$disk_free" -le 5242880 ]]; then
+ echo "LOW"
+else
+ echo "OK"
+fi
Property changes on: trunk/py/scripts/tools/bi_farm/new_node_check_space.sh
___________________________________________________________________
Added: svn:executable
+ *
More information about the Bf-extensions-cvs
mailing list