#
# Faiyaz Ahmed <faiyaza@cs.princeton.edu>
#
-# $Id: $
+# $Id: comon.py,v 1.4 2006/11/14 19:20:13 faiyaza Exp $
#
# Get CoMon data, unsorted, in CSV, and create a huge hash.
#
self.allbuckets.put(host)
def run(self):
- while 1:
- self.updatedb()
- self.updatebkts()
- self.push()
- time.sleep(COSLEEP)
+ self.updatedb()
+ self.updatebkts()
+ self.push()
def __repr__(self):
return self
for host in cdb.keys():
if cdb[host]['keyok'] == "0":
print("%s \t Bootstate %s nodetype %s kernver %s keyok %s" %(host, cdb[host]['bootstate'], cdb[host]['nodetype'], cdb[host]['kernver'], cdb[host]['keyok']))
+ print a.codata['michelangelo.ani.univie.ac.at']
#time.sleep(3)
#a.push()
#print a.filerw
#
# Faiyaz Ahmed <faiyaza@cs.princeton.edu>
#
-# $Id: emailTxt.py,v 1.4 2006/11/14 19:20:13 faiyaza Exp $
+# $Id: emailTxt.py,v 1.5 2007/01/10 20:08:44 faiyaza Exp $
#
#
class mailtxt:
- dbg=("""PlanetLab node %(hostname)s down.""", """As part of PlanetLab node monitoring, we noticed node %(hostname)s has been down for some time.
-Please check the node's connectivity and, if properly networked, power cycle the machine. If there are any console messages relating to the node's failure, please pass those to the PlanetLab-support mailing list so we can resolve the issue.
+ down=("""PlanetLab node %(hostname)s down.""", """As part of PlanetLab node monitoring, we noticed %(hostname)s has been down for %(days)s days.
+Please check the node's connectivity and, if properly networked, power cycle the machine. Note that rebooting the machine may not fully resolve the problems we're seeing. Once the machine has come back up, please visit the Comon status page (http://summer.cs.princeton.edu/status/tabulator.cgi?table=table_nodeviewshort&select='address==%(hostbyteorder)s') to verify that your node is accessible from the network.
+
+There's no need to respond to this message if CoMon reports that your machine is accessible. However, if there are any console messages relating to the node's failure, please report them to PlanetLab support (support@planet-lab.org) so we can resolve the issue.
Thanks.
+
-- PlanetLab Central (support@planet-lab.org)
""")
- down=("""PlanetLab node %(hostname)s down.""", """As part of PlanetLab node monitoring, we noticed node %(hostname)s has been down for some time.
-Please check the node's connectivity and, if properly networked, power cycle the machine. If there are any console messages relating to the node's failure, please pass those to the PlanetLab-support mailing list so we can resolve the issue.
+ dbg=("""Planetlab node %(hostname)s requires reboot.""", """As part of PlanetLab node monitoring, we noticed %(hostname)s is in debug mode. This usually implies the node was rebooted unexpectedly and could not come up cleanly.
+
+Please check the node's connectivity and, if properly networked, power cycle the machine. Note that rebooting the machine may not fully resolve the problems we're seeing. Once the machine has come back up, please visit the Comon status page (http://summer.cs.princeton.edu/status/tabulator.cgi?table=table_nodeviewshort&select='address==%(hostbyteorder)s') to verify that your node is accessible from the network.
+There's no need to respond to this message if CoMon reports that your machine is accessible. However, if there are any console messages relating to the node's failure, please report them to PlanetLab support (support@planet-lab.org) so we can resolve the issue.
Thanks.
+
-- PlanetLab Central (support@planet-lab.org)
""")
- ssh=("""Planetlab node %(hostname)s down.""", """As part of PlanetLab node monitoring, we noticed node %(hostname)s is not available for ssh.
-Please check the node's connectivity and, if properly networked, power cycle the machine. If there are any console messages relating to the node's failure, please pass those to the PlanetLab-support mailing list so we can resolve the issue.
+ ssh=("""Planetlab node %(hostname)s down.""", """As part of PlanetLab node monitoring, we noticed node %(hostname)s is not available for ssh.
+
+Please check the node's connectivity and, if properly networked, power cycle the machine. Note that rebooting the machine may not fully resolve the problems we're seeing. Once the machine has come back up, please visit the Comon status page (http://summer.cs.princeton.edu/status/tabulator.cgi?table=table_nodeviewshort&select='address==%(hostbyteorder)s') to verify that your node is accessible from the network.
+There's no need to respond to this message if CoMon reports that your machine is accessible. However, if there are any console messages relating to the node's failure, please report them to PlanetLab support (support@planet-lab.org) so we can resolve the issue.
Thanks.
+
-- PlanetLab Central (support@planet-lab.org)
""")
- dns=("""Planetlab node %(hostname)s down.""", """As part of PlanetLab node monitoring, we noticed the DNS servers used by %(hostname)s are not responding to queries.
+
+ dns=("""Planetlab node %(hostname)s down.""", """As part of PlanetLab node monitoring, we noticed the DNS servers used by %(hostname)s are not responding to queries.
Please verify the DNS information used by the node is correct. You can find directions on how to update the node's network information on the PlanetLab Technical Contacts Guid (http://www.planet-lab.org/doc/TechsGuide.php#id268898).
-- PlanetLab Central (support@planet-lab.org)
""")
- filerw=("""Planetlab node %(hostname)s has a bad disk.""", """As part of PlanetLab node monitoring, we noticed %(hostname)s has a read-only filesystem.
+
+ filerw=("""Planetlab node %(hostname)s has a bad disk.""", """As part of PlanetLab node monitoring, we noticed %(hostname)s has a read-only filesystem.
Please verify the integrity of the disk and email the site if a replacement is needed.
-- PlanetLab Central (support@planet-lab.org)
""")
- clock_drift=("""Planetlab node %(hostname)s and NTP.""", """As part of PlanetLab node monitoring, we noticed %(hostname)s cannot reach our NTP server.
+
+ clock_drift=("""Planetlab node %(hostname)s and NTP.""", """As part of PlanetLab node monitoring, we noticed %(hostname)s cannot reach our NTP server.
Please verify that the NTP port (tcp/123) is not blocked by your site.
-- PlanetLab Central (support@planet-lab.org)
""")
+
- dbg=("""Planetlab node %(hostname)s requires reboot.""", """As part of PlanetLab node monitoring, we noticed %(hostname)s is in debug mode. This usually implies the node was rebooted unexpectedly and could not come up cleanly.
-
-We have set the node to reinstall upon reboot. Please reboot the machine. It would be helpful if you could forward any error messages on the console to support@planet-lab.org.
-
-
-Thanks.
-
- -- PlanetLab Central (support@planet-lab.org)
-""")
-
-
- removedSliceCreation=("""PlanetLab slice creation/renewal suspension.""","""As part of PlanetLab node monitoring, we noticed the %(loginbase)s site has less than 2 nodes up. We have attempted to contact the PI and Technical contacts multiple times and have not received a response.
+ removedSliceCreation=("""PlanetLab slice creation/renewal suspension.""","""As part of PlanetLab node monitoring, we noticed the %(loginbase)s site has less than 2 nodes up. We have attempted to contact the PI and Technical contacts %(times)s times and have not received a response.
Slice creation and renewal are now suspended for the %(loginbase)s site. Please be aware that failure to respond will result in the automatic suspension of all running slices on PlanetLab.
-- PlanetLab Central (support@planet-lab.org)
""")
- suspendSlices=("""PlanetLab slices suspended.""","""As part of PlanetLab node monitoring, we noticed the %(loginbase)s site has less than 2 nodes up. We have attempted to contact the PI and Technical contacts multiple times and have not received a response.
+
+ suspendSlices=("""PlanetLab slices suspended.""","""As part of PlanetLab node monitoring, we noticed the %(loginbase)s site has less than 2 nodes up. We have attempted to contact the PI and Technical contacts %(times)s times and have not received a response.
All %(loginbase)s slices are now suspended.
""")
- STANDARD_PI="""As part of PlanetLab nodes monitoring, we noticed the node %(hostname)s is not available for ssh. We have made several attempts to contact the techinical contacts for this site (they are CCed) to help us bring the node back online. If there should be a different technical contact appointed, you may add the 'tech' role to any user registered for your site via the website. (Manage Users off the left nav bar on the PI tab, then click the user)
-
-Our records indicate that there is no remote power control unit connected to this node. If this is not the case, please log into the PlanetLab Website and update the PCU information.
+ pcu_broken=("""%(hostname)s failed to reinstall""","""Hello,
-https://www.planet-lab.org/db/pcu/
-
-Please check the machine's connectivity and, if properly networked, power cycle the machine to reboot it. If there are any console messages relating to the machines's failure, please pass those to the PlanetLab-support mailing list so we can resolve any problems.
+ %(hostname)s was remotely rebooted via your power control unit but has not contacted PlanetLab since. It should contact upon every boot, hence we believe that either the node has some hardware problems, is not properly connected to the power control unit, or has network connectivity issues. Could you please reboot the node and watch the console for error messages?
Thanks.
- -- PlanetLab Central (support@planet-lab.org)
-"""
-
- PCU_DOWN="""Hello,
-
-IMPORTANT: PLC has recently upgraded their monitoring system. One significant change is that PCU reboot attempts may now come from the following subnet:
-128.112.154.64/26
-If you have source IP filtering on your PCU please add this subnet.
-
-We attempted to reboot nodes at your site that appear to be down and were unable to connect to the power control unit. Could you please check and verify its network connectivity? Certain units benefit from a power cycle. Occasionally the problem is that othe network information for the PCU is incorrect on the PL website. If the unit seems fine, please verify that the information is correct by logging into the website and clicking the 'power control units' link in the lefthand navigation bar. Please let us know if experience any problems.
-
-Thanks.
-
- -- PlanetLab Central (support@planet-lab.org)
-"""
+-- PlanetLab Central (support@planet-lab.org)
+""")
- PCU_INEFFECTIVE="""Hello,
+ no_pcu=("""Hello,
-%(hostname)s was remotely rebooted via your power control unit but has not contacted PlanetLab since. It should contact upon every boot, hence we believe that either the node has some hardware problems, is not properly connected to the power control unit, or has network connectivity issues. Could you please reboot the node and watch the console for error messages? 'Couldn't resolve bootX.planet-lab.org' usualy mean connectivity problems. This could be either network configuration or occasionally filtering by the local network admins due to unusual traffic.
+We have set %(hostname)s to reinstall, but because your site does not have a power control unit, we are unable to powercycle the node. Please
Thanks.
- -- PlanetLab Central (support@planet-lab.org)
-"""
-
- PCU_INEFFECTIVE_PI="""Hello,
-
-We have made several attempts to contact the techinical contacts for this site (they are CCed) to help us bring the node back online. If there should be a different techinical contact appointed, you may add the 'tech' role to any user registered for your site via the website. (Manage Users off the left nav bar on the PI tab, then click the user)
-
-%(hostname)s was remotely rebooted via your power control unit but has not contacted PlanetLab since. It should contact upon every boot, hence we believe that either the node has some hardware problems, is not properly connected to the power control unit, or has network connectivity issues. Could you please reboot the node and watch the console for error messages? 'Couldn't resolve bootX.planet-lab.org' usually indicates connectivity problems. This could be either network configuration or occasionally filtering by the local network admins due to unusual traffic.
-
-Thanks.
+-- PlanetLab Central (support@planet-lab.org)
+""")
- -- PlanetLab Central (support@planet-lab.org)
-"""
#
# Faiyaz Ahmed <faiyaza@cs.princeton.edu>
#
-# $Id: $
+# $Id: monitor.py,v 1.4 2006/11/14 19:20:13 faiyaza Exp $
import sys
import os
# The meat of it.
# Nodes to check. Queue of all sick nodes.
- toCheck = Queue.Queue()
+ toCheck = Queue.Queue()
# Nodes that are sick w/o tickets
sickNoTicket = Queue.Queue()
# Comon DB of all nodes
# Nodes that are down. Use this to maintain DB; cleanup.
#alldown = Queue.Queue()
# RT DB
- tickets = {}
+ tickets = {}
# Nodes we've emailed.
# host - > (type of email, time)
emailed = {}
startThread(cm1,"comon")
# Wait for threads to init. Probably should join, but work on that later.
- time.sleep(10)
+ time.sleep(15)
+# while toCheck.empty() == false:
+# time.sleep(10)
+#
# Start Sending Emails
+ time.sleep(30)
startThread(pol, "policy")
-
- # Wait to finish
- while (sickNoTicket.empty() == False) or (toCheck.empty() == False):
- time.sleep(15)
-
-
+ time.sleep(10)
# Store state of emails
- pol.emailedStore("WRITE")
+# pol.emailedStore("WRITE")
# Email what we did.
- pol.status()
+# pol.status()
logger.info('Monitor Exitted')
#if not debug:
#
# Faiyaz Ahmed <faiyaza@cs.princeton.edu>
#
-# $Id: policy.py,v 1.11 2007/04/06 16:16:54 faiyaza Exp $
+# $Id: policy.py,v 1.12 2007/04/06 17:38:14 faiyaza Exp $
#
# Policy Engine.
#Thresholds (DAYS)
SPERDAY = 86400
PITHRESH = 7 * SPERDAY
-SLICETHRESH = 14 * SPERDAY
+SLICETHRESH = 7 * SPERDAY
# Days before attempting rins again
RINSTHRESH = 5 * SPERDAY
+# Days before calling the node dead.
+DEADTHRESH = 30 * SPERDAY
# Minimum number of nodes up before squeezing
MINUP = 2
# Email
# suspend slice creation
# kill slices
+
+
class Policy(Thread):
def __init__(self, comonthread, sickNoTicket, emailed):
self.cmn = comonthread
# host - > (time of email, type of email)
self.emailed = emailed
# all sick nodes w/o tickets
+ # from thread
self.sickNoTicket = sickNoTicket
- # Sitess we've Squeezed.
- self.squeezed = {}
+ # Actions taken on nodes.
+ # actionlogdb{node: [action, date]}
+ self.actionlogdb = {}
+ # Actions taken on sites.
+ # sitelogdb{site: [action, daysdown, date]}
+ self.sitelogdb = {}
+ # sick nodes with no tickets
+ # sickdb{loginbase: [{hostname1: [buckets]}, {...}]}
+ self.sickdb = {}
Thread.__init__(self)
-
- '''
- What to do when node is in dbg (as reported by CoMon).
- '''
+
+ def accumSickSites(self):
+ """
+ Take all sick nodes, find their sites, and put in
+ sickdb{loginbase: [{hostname1: [buckets]}, {...}]}
+ """
+ while self.sickNoTicket.empty() == False:
+ node = self.sickNoTicket.get(block = True)
+ bkts= []
+ for bkt in self.cmn.comonbkts.keys():
+ if (node in getattr(self.cmn, bkt)):
+ bkts.append("%s" % bkt)
+ self.sickdb[plc.siteId(node)] = {node: bkts}
+
+
def __actOnDebug(self, node):
- # Check to see if we've done this before
- if (node in self.emailed.keys()):
- if (self.emailed[node][0] == "dbg"):
- delta = time.time() - self.emailed[node][1]
- if (delta <= RINSTHRESH ):
- # Don't mess with node if under Thresh.
- # Return, move on.
- logger.info("POLICY: %s in dbg, but acted on %s days ago" % (node, delta // SPERDAY))
- return
- logger.info("POLICY: Node in dbg - " + node)
- plc.nodeBootState(node, "rins")
- # If it has a PCU
- return reboot.reboot(node)
-
- '''
- What to do when node is in dbg (as reported by CoMon).
- '''
+ """
+ If in debug, set the node to rins, reboot via PCU/POD
+ """
+ daysdown = self.cmn.codata[node]['sshstatus'] // (60*60*24)
+ logger.info("POLICY: Node %s in dbg. down for %s" %(node,daysdown))
+ plc.nodeBootState(node, "rins")
+ # If it has a PCU
+ reboot.reboot(node)
+ # Log it
+ self.actionlogdb[node] = ['rins', daysdown, time.time()]
+
+
+ def __actOnDown(self, node):
+ """
+ If down (not debug), do the same as actOnDebug for now
+ """
+ self.__actOnDebug(node)
+
+
def __actOnFilerw(self, node):
+ """
+ Report to PLC when node needs disk checked.
+ """
target = [PLCEMAIL]
logger.info("POLICY: Emailing PLC for " + node)
tmp = emailTxt.mailtxt.filerw
sbj = tmp[0] % {'hostname': node}
msg = tmp[1] % {'hostname': node}
mailer.email(sbj, msg, target)
- self.emailed[node] = ("filerw", time.time())
+ self.actionlogdb[node] = ["filerw", None, time.time()]
+
+
+ def __actOnDNS(self, node):
+ """
+ """
+
+
+ def __policy(self, node, loginbase, bkt):
+ # ...and spam 'em
+ target = [TECHEMAIL % loginbase]
+ tmp = emailTxt.mailtxt.down
+ sbj = tmp[0] % {'hostname': node}
+ msg = tmp[1] % {'hostname': node, 'days': daysdown}
+ mailer.email(sbj, msg, target)
+
+
- '''
- Acts on sick nodes.
- '''
def actOnSick(self):
- # Get list of nodes in debug from PLC
- #dbgNodes = NodesDebug()
+ """
+ Acts on sick nodes.
+ """
global TECHEMAIL, PIEMAIL
- # Grab a node from the queue (pushed by rt thread).
- node = self.sickNoTicket.get(block = True)
- # Get the login base
- loginbase = plc.siteId([node])
# Princeton Backdoor
if loginbase == "princeton": return
- # Send appropriate message for node if in appropriate bucket.
+ # Send appropriate message for node if in appropriate bucket.
# If we know where to send a message
if not loginbase:
logger.info("POLICY: loginbase for %s not found" %node)
# If in dbg, set to rins, then reboot. Inform PLC.
if (node in self.cmn.dbg):
- # If reboot failure via PCU, POD and send email
- # if contacted PCU, return
- if self.__actOnDebug(node): return
+ self.__actOnDebug(node)
if (node in self.emailed.keys()) and \
(node not in self.cmn.filerw) and \
delta // SPERDAY))
# If no luck with tech, email PI
- if (delta >= 1):
+ if (delta >= SPERDAY):
target.append(PIEMAIL % loginbase)
- # If more than PI thresh, but less than slicethresh
- if (delta >= PITHRESH) and (delta < SLICETHRESH):
+ if (delta >= 7 * SPERDAY):
#remove slice creation if enough nodes arent up
if not self.enoughUp(loginbase):
- slices = plc.slices([loginbase])
+ slices = plc.slices(loginbase)
if len(slices) >= 1:
for slice in slices:
target.append(SLICEMAIL % slice)
tmp = emailTxt.mailtxt.removedSliceCreation
sbj = tmp[0]
msg = tmp[1] % {'loginbase': loginbase}
- plc.removeSliceCreation([node])
+ plc.removeSliceCreation(node)
mailer.email(sbj, msg, target)
self.squeezed[loginbase] = (time.time(), "creation")
self.emailed[node] = ("creation", time.time())
%("creation", node, target))
return
- # If more than PI thresh and slicethresh
- if (delta >= PITHRESH) and (delta > SLICETHRESH):
+ if (delta >= 14 * SPERDAY):
target.append(PIEMAIL % loginbase)
# Email slices at site.
slices = plc.slices([loginbase])
return
- '''
+ """
Prints, logs, and emails status of up nodes, down nodes, and buckets.
- '''
+ """
def status(self):
sub = "Monitor Summary"
msg = "\nThe following nodes were acted upon: \n\n"
logger.info(msg)
return
- '''
+ """
Store/Load state of emails. When, where, what.
- '''
+ """
def emailedStore(self, action):
try:
if action == "LOAD":
except Exception, err:
logger.info("POLICY: Problem with DAT, %s" %err)
- '''
+ """
Returns True if more than MINUP nodes are up at a site.
- '''
+ """
def enoughUp(self, loginbase):
allsitenodes = plc.getSiteNodes([loginbase])
if len(allsitenodes) == 0:
def run(self):
- while 1:
- self.actOnSick()
- self.emailedStore("WRITE")
+ self.accumSickSites()
+ #self.actOnSick()
+ #self.emailedStore("WRITE")
+ print self.sickdb
+
def main():
#a.emailedStore("LOAD")
#print a.emailed
- print plc.slices([plc.siteId(["alice.cs.princeton.edu"])])
+ #print plc.slices([plc.siteId(["alice.cs.princeton.edu"])])
os._exit(0)
if __name__ == '__main__':
import os
git://git.onelab.eu / monitor.git / commitdiff