* Beginnings of slice freeze and node rins via PLC api. Still need to finish PCU stuff.
# Update individual buckekts. Hostnames only.
def updatebkts(self):
for (bkt,url) in self.comonbkts.items():
+ logger.debug("COMON: Updating bucket %s" % bkt)
tmp = self.coget(COMONURL + "&format=formatcsv&select='" + url + "'").keys()
setattr(self, bkt, tmp)
def coget(self,url):
rawdata = None
try:
- logger.debug("Trying - " + url)
coserv = urllib2.Request(url)
coserv.add_header('User-Agent',
'PL_Monitor +http://monitor.planet-lab.org/')
#
class mailtxt:
- DOWN="""As part of PlanetLab node monitoring, we noticed node %(hostname)s has been down for some time.
+ dbg=("""PlanetLab node %(hostname)s down.""", """As part of PlanetLab node monitoring, we noticed node %(hostname)s has been down for some time.
Please check the node's connectivity and, if properly networked, power cycle the machine. If there are any console messages relating to the node's failure, please pass those to the PlanetLab-support mailing list so we can resolve the issue.
Thanks.
-- PlanetLab Central (support@planet-lab.org)
-"""
+""")
- SSH="""As part of PlanetLab node monitoring, we noticed node %(hostname)s is not available for ssh.
+ down=("""PlanetLab node %(hostname)s down.""", """As part of PlanetLab node monitoring, we noticed node %(hostname)s has been down for some time.
Please check the node's connectivity and, if properly networked, power cycle the machine. If there are any console messages relating to the node's failure, please pass those to the PlanetLab-support mailing list so we can resolve the issue.
Thanks.
-- PlanetLab Central (support@planet-lab.org)
-"""
- DNS="""As part of PlanetLab node monitoring, we noticed the DNS servers used by %(hostname)s are not responding to queries.
+""")
+
+ ssh=("""Planetlab node %(hostname)s down.""", """As part of PlanetLab node monitoring, we noticed node %(hostname)s is not available for ssh.
+
+Please check the node's connectivity and, if properly networked, power cycle the machine. If there are any console messages relating to the node's failure, please pass those to the PlanetLab-support mailing list so we can resolve the issue.
+
+
+Thanks.
+
+ -- PlanetLab Central (support@planet-lab.org)
+""")
+
+ dns=("""Planetlab node %(hostname)s down.""", """As part of PlanetLab node monitoring, we noticed the DNS servers used by %(hostname)s are not responding to queries.
Please verify the DNS information used by the node is correct. You can find directions on how to update the node's network information on the PlanetLab Technical Contacts Guid (http://www.planet-lab.org/doc/TechsGuide.php#id268898).
Thanks.
-- PlanetLab Central (support@planet-lab.org)
-"""
- HDRO="""As part of PlanetLab node monitoring, we noticed %(hostname)s has a read-only filesystem.
+""")
+
+ filerw=("""Planetlab node %(hostname)s has a bad disk.""", """As part of PlanetLab node monitoring, we noticed %(hostname)s has a read-only filesystem.
-Please verify the disk is damaged and email the site if a replacement is needed.
+Please verify the integrity of the disk and email the site if a replacement is needed.
Thanks.
-- PlanetLab Central (support@planet-lab.org)
-"""
+""")
STANDARD_PI="""As part of PlanetLab nodes monitoring, we noticed the node %(hostname)s is not available for ssh. We have made several attempts to contact the techinical contacts for this site (they are CCed) to help us bring the node back online. If there should be a different technical contact appointed, you may add the 'tech' role to any user registered for your site via the website. (Manage Users off the left nav bar on the PI tab, then click the user)
128.112.154.64/26
If you have source IP filtering on your PCU please add this subnet.
-We have attempted to reboot some nodes at your site that appear to be down and found that we were unable to connect to the power control unit. Could you please bring it back online? Certain units benefit from a power cycle. Occasionally the problem is that our information for the PCU is incorrect. If the unit seems fine, please verify that the information is correct by logging into the website and clicking the 'power control units' link in the lefthand nav bar. Please let us know if you run into problems.
+We attempted to reboot nodes at your site that appear to be down and were unable to connect to the power control unit. Could you please check and verify its network connectivity? Certain units benefit from a power cycle. Occasionally the problem is that othe network information for the PCU is incorrect on the PL website. If the unit seems fine, please verify that the information is correct by logging into the website and clicking the 'power control units' link in the lefthand navigation bar. Please let us know if experience any problems.
Thanks.
-- PlanetLab Central (support@planet-lab.org)
"""
- PCU_DOWN_PI="""Hello,
-
-We have made several attempts to contact the techinical contacts for this site (they are CCed) to help us bring the node back online. If there should be a different technical contact appointed, you may add the 'tech' role to any user registered for your site via the website. (Manage Users off the left nav bar on the PI tab, then click the user)
-
-We have attempted to reboot some nodes at your site that appear to be down and found that we were unable to connect to the power control unit. Could you please bring it back online? Certain units benefit from a power cycle. Occasionally the problem is that our information for the PCU is incorrect. If the unit seems fine, please verify that the information is correct by logging into the website and clicking the 'power control units' link in the lefthand nav bar. Please let us know if you run into problems.
-
-Thanks.
-
- -- PlanetLab Central (support@planet-lab.org)
-"""
PCU_INEFFECTIVE="""Hello,
loginbase = api.AnonAdmGetSites (anon, site_id, ["login_base"])
return loginbase[0]['login_base']
-def email (subject, text, to, cc = None):
+
+def slices(loginbase):
+ api = xmlrpclib.Server(XMLRPC_SERVER, verbose=False)
+ return api.SliceListNames (auth.auth, loginbase)
+
+
+def email (subject, text, to):
"""Create a mime-message that will render HTML in popular
MUAs, text in better ones"""
import MimeWriter
# message body
#
writer.addheader("Subject", subject)
- writer.addheader("To", to)
- if cc: writer.addheader("CC", cc)
+ if to.__class__ == [].__class__ :
+ writer.addheader("To", to[0])
+ cc = ""
+ for dest in to[1:len(to)]:
+ cc +="%s, " % dest
+ cc = cc.rstrip(", ")
+ writer.addheader("CC", cc)
+ else:
+ writer.addheader("To", to)
+
writer.addheader("MIME-Version", "1.0")
#
# start the multipart section of the message
writer.lastpart()
msg = out.getvalue()
out.close()
-
- #server = smtplib.SMTP(MTA)
- #server.sendmail(FROM, (to,cc), msg)
- #server.quit()
+ server = smtplib.SMTP(MTA)
+ server.sendmail(FROM, to, msg)
+ server.quit()
if __name__=="__main__":
import smtplib
import emailTxt
- id = siteId("alice.cs.princeeton.edu")
- if id:
- email('TEST', emailTxt.mailtxt.STANDARD % {'hostname': "ALICE.cs.princeton.edu"}, "tech-" + id + "@sites.planet-lab.org")
- else:
- print "No dice."
+ id = siteId("alice.cs.princeton.edu")
+ print id
+ #if id:
+ #email('TEST', emailTxt.mailtxt.ssh % {'hostname': "ALICE.cs.princeton.edu"}, "tech-" + id + "@sites.planet-lab.org")
+ #else:
+ # print "No dice."
+ #email("TEST109", "THIS IS A TEST", ["faiyaza@cs.princeton.edu", "faiyaz@winlab.rutgers.edu", "faiyaza@gmail.com"])
# Email defaults
MTA="localhost"
FROM="support@planet-lab.org"
+TECHEMAIL="tech-%s@sites.planet-lab.org"
+PIEMAIL="pi-%s@sites.planet-lab.org"
# API
XMLRPC_SERVER = 'https://www.planet-lab.org/PLCAPI/'
# Wait for threads to init. Probably should join, but work on that later.
time.sleep(10)
+
# Start Sending Emails
startThread(pol, "policy")
time.sleep(15)
+
+ pol.status()
+
+ # Store state of emails
pol.emailedStore("WRITE")
+
+ # Email what we did.
+ pol.status()
+
logger.info('Monitor Exitted')
#if not debug:
# removepid("monitor")
import xml, xmlrpclib
import Queue
+#Hack to auth structure
+import auth
DAT="./monitor.dat"
logger = logging.getLogger("monitor")
# Days between emails (enforce 'squeeze' after this time).
SQUEEZE = 3
+
+# Where to email the summary
+SUMTO = "faiyaza@cs.princeton.edu"
+TECHEMAIL="tech-%s@sites.planet-lab.org"
+PIEMAIL="pi-%s@sites.planet-lab.org"
+SLICEMAIL="%s@slices.planet-lab.org"
+PLCEMAIL="support@planet-lab.org"
+
+#Thresholds
+PITHRESH = 3
+SLICETHRESH = 5
+
# IF:
# no SSH, down.
# bad disk, down
'''
Acts on sick nodes
'''
- def emailsick(self):
+ def actOnSick(self):
# Get list of nodes in debug from PLC
#dbgNodes = NodesDebug()
-
+ global TECHEMAIL, PIEMAIL
node = self.sickNoTicket.get(block = True)
# Get the login base
id = mailer.siteId(node)
+ # Send appropriate message for node if in appropriate bucket.
+ # If we know where to send a message
if not id:
logger.info("loginbase for %s not found" %node)
- elif node not in self.emailed.keys():
- # Email about Down.
- if node in self.cmn.down:
- logger.debug("POLICY: Emailing (down) " + node)
- self.emailed[node] = ("down", time.localtime())
- msg = emailTxt.mailtxt.DOWN \
- % {'hostname': node}
- mailer.email(node + " down", msg,
- "tech-" + id + "@sites.planet-lab.org")
- return
-
- # Email about no SSH.
- if node in self.cmn.ssh:
- logger.debug("POLICY: Emailing (ssh) " + node)
- self.emailed[node] = ("ssh", time.localtime())
- msg = emailTxt.mailtxt.SSH \
- % {'hostname': node}
- mailer.email(node + " down", msg,
- "tech-" + id + "@sites.planet-lab.org")
- return
-
- # Email about DNS
- if node in self.cmn.dns:
- logger.debug("POLICY: Emailing (dns)" + node)
- self.emailed[node] = ("dns", time.localtime())
- msg = emailTxt.mailtxt.DNS \
- % {'hostname': node}
- mailer.email("Please update DNS used by " \
- + node, msg,
- "tech-" + id + "@sites.planet-lab.org")
- return
-
+ # And we didn't email already.
+ else:
+ # If first email, send to Tech
+ target = [TECHEMAIL % id]
+
+ # If disk is foobarred, PLC should check it.
+ if (node in self.cmn.filerw) and \
+ (node not in self.emailed.keys()):
+ target = [PLCEMAIL]
+ logger.info("Emailing PLC for " + node)
+
+ # If in dbg, set to rins, then reboot. Inform PLC.
+ if (node in self.cmn.dbg):
+ logger.info("Node in dbg - " + node)
+ return
+
+ # If its a disk, email PLC; dont bother going through this loop.
+ if (node in self.emailed.keys()) and \
+ (node not in self.cmn.filerw):
+ # If we emailed before, how long ago?
+ delta = time.localtime()[2] - self.emailed[node][1][2]
+ # If more than PI thresh, but less than slicethresh
+ if (delta >= PITHRESH) and (delta < SLICETHRESH):
+ logger.info("Emailing PI for " + node)
+ target.append(PIEMAIL % id)
+ # If more than PI thresh and slicethresh
+ if (delta >= PITHRESH) and (delta > SLICETHRESH):
+ logger.info("Emailing slices for " + node)
+ # Email slices at site.
+ slices = mailer.slices(id)
+ if len(slices) >= 1:
+ for slice in slices:
+ target.append(SLICEMAIL % slice)
+
+ # Find the bucket the node is in and send appropriate email
+ # to approriate list of people.
+ for bkt in self.cmn.comonbkts.keys():
+ if (node in getattr(self.cmn, bkt)):
+ # Send predefined message for that bucket.
+ logger.info("POLICY: Emailing (%s) %s - %s"\
+ %(bkt, node, target))
+ tmp = getattr(emailTxt.mailtxt, bkt)
+ sbj = tmp[0] % {'hostname': node}
+ msg = tmp[1] % {'hostname': node}
+ mailer.email(sbj, msg, target)
+ self.emailed[node] = (bkt , time.localtime())
+ return
+
'''
Prints, logs, and emails status of up nodes, down nodes, and buckets.
'''
def status(self):
- return 0
+ sub = "Monitor Summary"
+ msg = "\nThe following nodes were acted upon: \n\n"
+ for (node, (type, date)) in self.emailed.items():
+ msg +="%s\t(%s)\t%s:%s:%s\n" %(node,type,date[3],date[4],date[5])
+ mailer.email(sub, msg, [SUMTO])
+ logger.info(msg)
+ return
'''
Store/Load state of emails. When, where, what.
self.emailed.update(pickle.load(f))
if action == "WRITE":
f = open(DAT, "w")
- logger.info("Writing " + DAT)
+ logger.debug("Writing " + DAT)
pickle.dump(self.emailed, f)
f.close()
except Exception, err:
def run(self):
while 1:
- self.emailsick()
-
+ self.actOnSick()
+ self.emailedStore("WRITE")
'''
Returns list of nodes in dbg as reported by PLC
'''
logger.addHandler(ch)
#print NodesDebug()
- tmp = Queue.Queue()
- a = Policy(None, tmp)
- a.emailedStore("LOAD")
- print a.emailed
+ #tmp = Queue.Queue()
+ #a = Policy(None, tmp)
+ #a.emailedStore("LOAD")
+ #print a.emailed
+ print siteId("princetoan")
os._exit(0)
if __name__ == '__main__':
git://git.onelab.eu / monitor.git / commitdiff