#!/usr/bin/python
-import soltesz
+from monitor import database
import plc
-import auth
-api = plc.PLC(auth.auth, auth.plc)
+api = plc.getAuthAPI()
-import config
import mailer
import time
+from model import *
+from const import *
+import util.file
+import config
+
def gethostlist(hostlist_file):
- return config.getListFromFile(hostlist_file)
+ return util.file.getListFromFile(hostlist_file)
#nodes = api.GetNodes({'peer_id' : None}, ['hostname'])
#return [ n['hostname'] for n in nodes ]
# condition/penalty is applied, move to the next phase.
-fb = soltesz.dbLoad("findbad")
+#fb = database.dbLoad("findbad")
class RT(object):
def __init__(self, ticket_id = None):
class Recent(object):
def __init__(self, withintime):
self.withintime = withintime
- self.time = time.time()
- self.action_taken = False
+
+ try:
+ self.time = self.__getattribute__('time')
+ except:
+ self.time = time.time()- 7*24*60*60
+
+ #self.time = time.time()
+ #self.action_taken = False
def isRecent(self):
if self.time + self.withintime < time.time():
db = "persistflags"
try:
- pm = soltesz.dbLoad(db)
+ pm = database.dbLoad(db)
except:
- soltesz.dbDump(db, {})
- pm = soltesz.dbLoad(db)
+ database.dbDump(db, {})
+ pm = database.dbLoad(db)
#print pm
if id in pm:
obj = pm[id]
obj = super(PersistFlags, typ).__new__(typ, *args, **kwargs)
for key in kwargs.keys():
obj.__setattr__(key, kwargs[key])
+ obj.time = time.time()
+ obj.action_taken = False
obj.db = db
return obj
Recent.__init__(self, withintime)
def save(self):
- pm = soltesz.dbLoad(self.db)
+ pm = database.dbLoad(self.db)
pm[self.id] = self
- soltesz.dbDump(self.db, pm)
+ database.dbDump(self.db, pm)
def resetFlag(self, name):
self.__setattr__(name, False)
self.__setattr__(name, False)
return False
+ def resetRecentFlag(self, name):
+ self.resetFlag(name)
+ self.unsetRecent()
+
def setRecentFlag(self, name):
self.setFlag(name)
self.setRecent()
self.__setattr__(name, False)
return False
+ def checkattr(self, name):
+ try:
+ x = self.__getattribute__(name)
+ return True
+ except:
+ return False
+
+
class PersistMessage(Message):
def __new__(typ, id, subject, message, via_rt, **kwargs):
if 'db' in kwargs:
db = "persistmessages"
try:
- pm = soltesz.dbLoad(db)
+ pm = database.dbLoad(db)
except:
- soltesz.dbDump(db, {})
- pm = soltesz.dbLoad(db)
+ database.dbDump(db, {})
+ pm = database.dbLoad(db)
#print pm
if id in pm:
obj.actiontracker = Recent(3*60*60*24)
obj.ticket_id = None
+ if 'ticket_id' in kwargs and kwargs['ticket_id'] is not None:
+ obj.ticket_id = kwargs['ticket_id']
+
obj.db = db
return obj
self.actiontracker.setRecent()
#print "recording object for persistance"
- pm = soltesz.dbLoad(self.db)
+ pm = database.dbLoad(self.db)
pm[self.id] = self
- soltesz.dbDump(self.db, pm)
+ database.dbDump(self.db, pm)
else:
# NOTE: only send a new message every week, regardless.
- print "Not sending to host b/c not within window of 6 days"
- pass
+ print "Not sending to host b/c not within window of %s days" % (self.actiontracker.withintime // 60*60*24)
class MonitorMessage(object):
def __new__(typ, id, *args, **kwargs):
try:
if 'reset' in kwargs and kwargs['reset'] == True:
- soltesz.dbDump(db, {})
- pm = soltesz.dbLoad(db)
+ database.dbDump(db, {})
+ pm = database.dbLoad(db)
except:
- soltesz.dbDump(db, {})
- pm = soltesz.dbLoad(db)
+ database.dbDump(db, {})
+ pm = database.dbLoad(db)
#print pm
if id in pm:
try:
if 'reset' in kwargs and kwargs['reset'] == True:
- soltesz.dbDump(db, {})
- pm = soltesz.dbLoad(db)
+ database.dbDump(db, {})
+ pm = database.dbLoad(db)
except:
- soltesz.dbDump(db, {})
- pm = soltesz.dbLoad(db)
+ database.dbDump(db, {})
+ pm = database.dbLoad(db)
#print pm
if id in pm:
def __init__(self, id, index, **kwargs):
self.id = id
- #SitePenalty.__init__(self, self.index)
def save(self):
- pm = soltesz.dbLoad(self.db)
+ pm = database.dbLoad(self.db)
pm[self.id] = self
- soltesz.dbDump(self.db, pm)
-
+ database.dbDump(self.db, pm)
class Target:
return con_or_true
+class Record(object):
+
+ def __init__(self, hostname, data):
+ self.hostname = hostname
+ self.data = data
+ self.plcdb_hn2lb = database.dbLoad("plcdb_hn2lb")
+ self.loginbase = self.plcdb_hn2lb[self.hostname]
+ return
+
+
+ def stageIswaitforever(self):
+ if 'waitforever' in self.data['stage']:
+ return True
+ else:
+ return False
+
+ def severity(self):
+ category = self.data['category']
+ prev_category = self.data['prev_category']
+ val = cmpCategoryVal(category, prev_category)
+ return val
+
+ def improved(self):
+ return self.severity() > 0
+
+ def end_record(self):
+ return node_end_record(self.hostname)
+
+ def reset_stage(self):
+ self.data['stage'] = 'findbad'
+ return True
+
+ def getCategory(self):
+ return self.data['category'].lower()
+
+ def getState(self):
+ return self.data['state'].lower()
+
+ def getDaysDown(cls, diag_record):
+ daysdown = -1
+ if diag_record['comonstats']['uptime'] != "null":
+ daysdown = - int(float(diag_record['comonstats']['uptime'])) // (60*60*24)
+ #elif diag_record['comonstats']['sshstatus'] != "null":
+ # daysdown = int(diag_record['comonstats']['sshstatus']) // (60*60*24)
+ #elif diag_record['comonstats']['lastcotop'] != "null":
+ # daysdown = int(diag_record['comonstats']['lastcotop']) // (60*60*24)
+ else:
+ now = time.time()
+ last_contact = diag_record['plcnode']['last_contact']
+ if last_contact == None:
+ # the node has never been up, so give it a break
+ daysdown = -1
+ else:
+ diff = now - last_contact
+ daysdown = diff // (60*60*24)
+ return daysdown
+ getDaysDown = classmethod(getDaysDown)
+
+ def getStrDaysDown(cls, diag_record):
+ daysdown = "unknown"
+ last_contact = diag_record['plcnode']['last_contact']
+ date_created = diag_record['plcnode']['date_created']
+
+ if diag_record['comonstats']['uptime'] != "null" and \
+ diag_record['comonstats']['uptime'] != "-1":
+ daysdown = int(float(diag_record['comonstats']['uptime'])) // (60*60*24)
+ daysdown = "%d days up" % daysdown
+
+ elif last_contact is None:
+ if date_created is not None:
+ now = time.time()
+ diff = now - date_created
+ daysdown = diff // (60*60*24)
+ daysdown = "Never contacted PLC, created %s days ago" % daysdown
+ else:
+ daysdown = "Never contacted PLC"
+ else:
+ now = time.time()
+ diff = now - last_contact
+ daysdown = diff // (60*60*24)
+ daysdown = "%s days down" % daysdown
+ return daysdown
+ getStrDaysDown = classmethod(getStrDaysDown)
+
+ #def getStrDaysDown(cls, diag_record):
+ # daysdown = cls.getDaysDown(diag_record)
+ # if daysdown > 0:
+ # return "%d days down"%daysdown
+ # elif daysdown == -1:
+ # return "Never online"
+ # else:
+ # return "%d days up"% -daysdown
+ #getStrDaysDown = classmethod(getStrDaysDown)
+
+ def takeAction(self):
+ pp = PersistSitePenalty(self.hostname, 0, db='persistpenalty_hostnames')
+ if 'improvement' in self.data['stage'] or self.improved():
+ print "decreasing penalty for %s"%self.hostname
+ pp.decrease()
+ else:
+ print "increasing penalty for %s"%self.hostname
+ pp.increase()
+ pp.apply(self.hostname)
+ pp.save()
+
+ def _format_diaginfo(self):
+ info = self.data['info']
+ if self.data['stage'] == 'monitor-end-record':
+ hlist = " %s went from '%s' to '%s'\n" % (info[0], info[1], info[2])
+ else:
+ hlist = " %s %s - %s\n" % (info[0], info[2], info[1]) #(node,ver,daysdn)
+ return hlist
+
+ def getMessage(self, ticket_id=None):
+ self.data['args']['hostname'] = self.hostname
+ self.data['args']['loginbase'] = self.loginbase
+ self.data['args']['hostname_list'] = self._format_diaginfo()
+ message = PersistMessage(self.hostname,
+ self.data['message'][0] % self.data['args'],
+ self.data['message'][1] % self.data['args'],
+ True, db='monitor_persistmessages',
+ ticket_id=ticket_id)
+ return message
+
+ def getContacts(self):
+ roles = self.data['email']
+
+ if not config.mail and not config.debug and config.bcc:
+ roles = ADMIN
+ if config.mail and config.debug:
+ roles = ADMIN
+
+ # build targets
+ contacts = []
+ if ADMIN & roles:
+ contacts += [config.email]
+ if TECH & roles:
+ contacts += [TECHEMAIL % self.loginbase]
+ if PI & roles:
+ contacts += [PIEMAIL % self.loginbase]
+ if USER & roles:
+ slices = plc.slices(self.loginbase)
+ if len(slices) >= 1:
+ for slice in slices:
+ contacts += [SLICEMAIL % slice]
+ print "SLIC: %20s : %d slices" % (self.loginbase, len(slices))
+ else:
+ print "SLIC: %20s : 0 slices" % self.loginbase
+
+ return contacts
+
+
class NodeRecord:
def __init__(self, hostname, target):
self.hostname = hostname
- self.pcu = PCU(hostname)
self.ticket = None
self.target = target
- if hostname in fb['nodes']:
- self.data = fb['nodes'][hostname]['values']
+ #if hostname in fb['nodes']:
+ # self.data = fb['nodes'][hostname]['values']
+ #else:
+ # raise Exception("Hostname not in scan database")
+
+ def stageIswaitforever(self):
+ if 'waitforever' in self.data['stage']:
+ return True
else:
- raise Exception("Hostname not in scan database")
+ return False
- def get(self):
- pass
def severity(self):
category = self.data['category']
prev_category = self.data['prev_category']
val = cmpCategoryVal(category, prev_category)
return val
+
+ def improved(self):
+ return self.severity() > 0
+
+ def end_record(self):
+ return node_end_record(self.hostname)
+
+ def reset_stage(self):
+ self.data['stage'] = 'findbad'
+ return True
+
def open_tickets(self):
if self.ticket and self.ticket.status['status'] == 'open':
return 1
self.ticket.closeTicket()
def exempt_from_penalties(self):
- bl = soltesz.dbLoad("l_blacklist")
+ bl = database.dbLoad("l_blacklist")
return self.hostname in bl
def penalties(self):
def _get_contacts_for_condition(self):
pass
+class Action(MonRecord):
+ def __init__(self, host, data):
+ self.host = host
+ MonRecord.__init__(self, data)
+ return
+
+ def deltaDays(self, delta):
+ t = datetime.fromtimestamp(self.__dict__['time'])
+ d = t + timedelta(delta)
+ self.__dict__['time'] = time.mktime(d.timetuple())
+
+def node_end_record(node):
+ act_all = database.dbLoad("act_all")
+ if node not in act_all:
+ del act_all
+ return False
+
+ if len(act_all[node]) == 0:
+ del act_all
+ return False
+
+ a = Action(node, act_all[node][0])
+ a.delField('rt')
+ a.delField('found_rt_ticket')
+ a.delField('second-mail-at-oneweek')
+ a.delField('second-mail-at-twoweeks')
+ a.delField('first-found')
+ rec = a.get()
+ rec['action'] = ["close_rt"]
+ rec['category'] = "UNKNOWN"
+ rec['stage'] = "monitor-end-record"
+ rec['time'] = time.time() - 7*60*60*24
+ act_all[node].insert(0,rec)
+ database.dbDump("act_all", act_all)
+ del act_all
+ return True
+
if __name__ == "__main__":
#r = RT()
- #r.email("test", "body of test message", ['soltesz@cs.princeton.edu'])
- from emailTxt import mailtxt
- soltesz.dbDump("persistmessages", {});
- args = {'url_list': 'http://www.planet-lab.org/bootcds/planet1.usb\n','hostname': 'planet1','hostname_list': ' blahblah - days down\n'}
- m = PersistMessage("blue", "test 1", mailtxt.newdown_one[1] % args, True)
- m.send(['soltesz@cs.utk.edu'])
- m = PersistMessage("blue", "test 1 - part 2", mailtxt.newalphacd_one[1] % args, True)
+ #r.email("test", "body of test message", ['database@cs.princeton.edu'])
+ #from emailTxt import mailtxt
+ print "loaded"
+ #database.dbDump("persistmessages", {});
+ #args = {'url_list': 'http://www.planet-lab.org/bootcds/planet1.usb\n','hostname': 'planet1','hostname_list': ' blahblah - days down\n'}
+ #m = PersistMessage("blue", "test 1", mailtxt.newdown_one[1] % args, True)
+ #m.send(['soltesz@cs.utk.edu'])
+ #m = PersistMessage("blue", "test 1 - part 2", mailtxt.newalphacd_one[1] % args, True)
# TRICK timer to thinking some time has passed.
- m.actiontracker.time = time.time() - 6*60*60*24
- m.send(['soltesz@cs.utk.edu'])
+ #m.actiontracker.time = time.time() - 6*60*60*24
+ #m.send(['soltesz@cs.utk.edu'])