X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=unified_model.py;h=acc89d8a09dc90d69c6a30f03b172806eec0be93;hb=944d143a6528c4157b71f51ed480aec806cbaa06;hp=918f653f128b01c894c5cb754fe8417d2d1a4dd6;hpb=590ac12c941310b40a92d2fe938e62e3538f2893;p=monitor.git diff --git a/unified_model.py b/unified_model.py index 918f653..acc89d8 100755 --- a/unified_model.py +++ b/unified_model.py @@ -1,17 +1,20 @@ #!/usr/bin/python -import soltesz +from monitor import database import plc -import auth -api = plc.PLC(auth.auth, auth.plc) +api = plc.getAuthAPI() -import config import mailer import time +from model import * +from const import * +import util.file +import config + def gethostlist(hostlist_file): - return config.getListFromFile(hostlist_file) + return util.file.getListFromFile(hostlist_file) #nodes = api.GetNodes({'peer_id' : None}, ['hostname']) #return [ n['hostname'] for n in nodes ] @@ -67,7 +70,7 @@ class PenaltyMap: # condition/penalty is applied, move to the next phase. -fb = soltesz.dbLoad("findbad") +#fb = database.dbLoad("findbad") class RT(object): def __init__(self, ticket_id = None): @@ -110,8 +113,14 @@ class Message(object): class Recent(object): def __init__(self, withintime): self.withintime = withintime - self.time = time.time() - self.action_taken = False + + try: + self.time = self.__getattribute__('time') + except: + self.time = time.time()- 7*24*60*60 + + #self.time = time.time() + #self.action_taken = False def isRecent(self): if self.time + self.withintime < time.time(): @@ -141,10 +150,10 @@ class PersistFlags(Recent): db = "persistflags" try: - pm = soltesz.dbLoad(db) + pm = database.dbLoad(db) except: - soltesz.dbDump(db, {}) - pm = soltesz.dbLoad(db) + database.dbDump(db, {}) + pm = database.dbLoad(db) #print pm if id in pm: obj = pm[id] @@ -152,6 +161,8 @@ class PersistFlags(Recent): obj = super(PersistFlags, typ).__new__(typ, *args, **kwargs) for key in kwargs.keys(): obj.__setattr__(key, kwargs[key]) + obj.time = time.time() + obj.action_taken = False obj.db = db return obj @@ -161,9 +172,9 @@ class PersistFlags(Recent): Recent.__init__(self, withintime) def save(self): - pm = soltesz.dbLoad(self.db) + pm = database.dbLoad(self.db) pm[self.id] = self - soltesz.dbDump(self.db, pm) + database.dbDump(self.db, pm) def resetFlag(self, name): self.__setattr__(name, False) @@ -178,6 +189,10 @@ class PersistFlags(Recent): self.__setattr__(name, False) return False + def resetRecentFlag(self, name): + self.resetFlag(name) + self.unsetRecent() + def setRecentFlag(self, name): self.setFlag(name) self.setRecent() @@ -191,6 +206,14 @@ class PersistFlags(Recent): self.__setattr__(name, False) return False + def checkattr(self, name): + try: + x = self.__getattribute__(name) + return True + except: + return False + + class PersistMessage(Message): def __new__(typ, id, subject, message, via_rt, **kwargs): if 'db' in kwargs: @@ -199,10 +222,10 @@ class PersistMessage(Message): db = "persistmessages" try: - pm = soltesz.dbLoad(db) + pm = database.dbLoad(db) except: - soltesz.dbDump(db, {}) - pm = soltesz.dbLoad(db) + database.dbDump(db, {}) + pm = database.dbLoad(db) #print pm if id in pm: @@ -215,6 +238,9 @@ class PersistMessage(Message): obj.actiontracker = Recent(3*60*60*24) obj.ticket_id = None + if 'ticket_id' in kwargs and kwargs['ticket_id'] is not None: + obj.ticket_id = kwargs['ticket_id'] + obj.db = db return obj @@ -232,13 +258,12 @@ class PersistMessage(Message): self.actiontracker.setRecent() #print "recording object for persistance" - pm = soltesz.dbLoad(self.db) + pm = database.dbLoad(self.db) pm[self.id] = self - soltesz.dbDump(self.db, pm) + database.dbDump(self.db, pm) else: # NOTE: only send a new message every week, regardless. - print "Not sending to host b/c not within window of 6 days" - pass + print "Not sending to host b/c not within window of %s days" % (self.actiontracker.withintime // 60*60*24) class MonitorMessage(object): def __new__(typ, id, *args, **kwargs): @@ -249,11 +274,11 @@ class MonitorMessage(object): try: if 'reset' in kwargs and kwargs['reset'] == True: - soltesz.dbDump(db, {}) - pm = soltesz.dbLoad(db) + database.dbDump(db, {}) + pm = database.dbLoad(db) except: - soltesz.dbDump(db, {}) - pm = soltesz.dbLoad(db) + database.dbDump(db, {}) + pm = database.dbLoad(db) #print pm if id in pm: @@ -321,11 +346,11 @@ class PersistSitePenalty(SitePenalty): try: if 'reset' in kwargs and kwargs['reset'] == True: - soltesz.dbDump(db, {}) - pm = soltesz.dbLoad(db) + database.dbDump(db, {}) + pm = database.dbLoad(db) except: - soltesz.dbDump(db, {}) - pm = soltesz.dbLoad(db) + database.dbDump(db, {}) + pm = database.dbLoad(db) #print pm if id in pm: @@ -342,13 +367,11 @@ class PersistSitePenalty(SitePenalty): def __init__(self, id, index, **kwargs): self.id = id - #SitePenalty.__init__(self, self.index) def save(self): - pm = soltesz.dbLoad(self.db) + pm = database.dbLoad(self.db) pm[self.id] = self - soltesz.dbDump(self.db, pm) - + database.dbDump(self.db, pm) class Target: @@ -385,24 +408,190 @@ class Target: return con_or_true +class Record(object): + + def __init__(self, hostname, data): + self.hostname = hostname + self.data = data + self.plcdb_hn2lb = database.dbLoad("plcdb_hn2lb") + self.loginbase = self.plcdb_hn2lb[self.hostname] + return + + + def stageIswaitforever(self): + if 'waitforever' in self.data['stage']: + return True + else: + return False + + def severity(self): + category = self.data['category'] + prev_category = self.data['prev_category'] + val = cmpCategoryVal(category, prev_category) + return val + + def improved(self): + return self.severity() > 0 + + def end_record(self): + return node_end_record(self.hostname) + + def reset_stage(self): + self.data['stage'] = 'findbad' + return True + + def getCategory(self): + return self.data['category'].lower() + + def getState(self): + return self.data['state'].lower() + + def getDaysDown(cls, diag_record): + daysdown = -1 + if diag_record['comonstats']['uptime'] != "null": + daysdown = - int(float(diag_record['comonstats']['uptime'])) // (60*60*24) + #elif diag_record['comonstats']['sshstatus'] != "null": + # daysdown = int(diag_record['comonstats']['sshstatus']) // (60*60*24) + #elif diag_record['comonstats']['lastcotop'] != "null": + # daysdown = int(diag_record['comonstats']['lastcotop']) // (60*60*24) + else: + now = time.time() + last_contact = diag_record['plcnode']['last_contact'] + if last_contact == None: + # the node has never been up, so give it a break + daysdown = -1 + else: + diff = now - last_contact + daysdown = diff // (60*60*24) + return daysdown + getDaysDown = classmethod(getDaysDown) + + def getStrDaysDown(cls, diag_record): + daysdown = "unknown" + last_contact = diag_record['plcnode']['last_contact'] + date_created = diag_record['plcnode']['date_created'] + + if diag_record['comonstats']['uptime'] != "null" and \ + diag_record['comonstats']['uptime'] != "-1": + daysdown = int(float(diag_record['comonstats']['uptime'])) // (60*60*24) + daysdown = "%d days up" % daysdown + + elif last_contact is None: + if date_created is not None: + now = time.time() + diff = now - date_created + daysdown = diff // (60*60*24) + daysdown = "Never contacted PLC, created %s days ago" % daysdown + else: + daysdown = "Never contacted PLC" + else: + now = time.time() + diff = now - last_contact + daysdown = diff // (60*60*24) + daysdown = "%s days down" % daysdown + return daysdown + getStrDaysDown = classmethod(getStrDaysDown) + + #def getStrDaysDown(cls, diag_record): + # daysdown = cls.getDaysDown(diag_record) + # if daysdown > 0: + # return "%d days down"%daysdown + # elif daysdown == -1: + # return "Never online" + # else: + # return "%d days up"% -daysdown + #getStrDaysDown = classmethod(getStrDaysDown) + + def takeAction(self): + pp = PersistSitePenalty(self.hostname, 0, db='persistpenalty_hostnames') + if 'improvement' in self.data['stage'] or self.improved(): + print "decreasing penalty for %s"%self.hostname + pp.decrease() + else: + print "increasing penalty for %s"%self.hostname + pp.increase() + pp.apply(self.hostname) + pp.save() + + def _format_diaginfo(self): + info = self.data['info'] + if self.data['stage'] == 'monitor-end-record': + hlist = " %s went from '%s' to '%s'\n" % (info[0], info[1], info[2]) + else: + hlist = " %s %s - %s\n" % (info[0], info[2], info[1]) #(node,ver,daysdn) + return hlist + + def getMessage(self, ticket_id=None): + self.data['args']['hostname'] = self.hostname + self.data['args']['loginbase'] = self.loginbase + self.data['args']['hostname_list'] = self._format_diaginfo() + message = PersistMessage(self.hostname, + self.data['message'][0] % self.data['args'], + self.data['message'][1] % self.data['args'], + True, db='monitor_persistmessages', + ticket_id=ticket_id) + return message + + def getContacts(self): + roles = self.data['email'] + + if not config.mail and not config.debug and config.bcc: + roles = ADMIN + if config.mail and config.debug: + roles = ADMIN + + # build targets + contacts = [] + if ADMIN & roles: + contacts += [config.email] + if TECH & roles: + contacts += [TECHEMAIL % self.loginbase] + if PI & roles: + contacts += [PIEMAIL % self.loginbase] + if USER & roles: + slices = plc.slices(self.loginbase) + if len(slices) >= 1: + for slice in slices: + contacts += [SLICEMAIL % slice] + print "SLIC: %20s : %d slices" % (self.loginbase, len(slices)) + else: + print "SLIC: %20s : 0 slices" % self.loginbase + + return contacts + + class NodeRecord: def __init__(self, hostname, target): self.hostname = hostname - self.pcu = PCU(hostname) self.ticket = None self.target = target - if hostname in fb['nodes']: - self.data = fb['nodes'][hostname]['values'] + #if hostname in fb['nodes']: + # self.data = fb['nodes'][hostname]['values'] + #else: + # raise Exception("Hostname not in scan database") + + def stageIswaitforever(self): + if 'waitforever' in self.data['stage']: + return True else: - raise Exception("Hostname not in scan database") + return False - def get(self): - pass def severity(self): category = self.data['category'] prev_category = self.data['prev_category'] val = cmpCategoryVal(category, prev_category) return val + + def improved(self): + return self.severity() > 0 + + def end_record(self): + return node_end_record(self.hostname) + + def reset_stage(self): + self.data['stage'] = 'findbad' + return True + def open_tickets(self): if self.ticket and self.ticket.status['status'] == 'open': return 1 @@ -419,7 +608,7 @@ class NodeRecord: self.ticket.closeTicket() def exempt_from_penalties(self): - bl = soltesz.dbLoad("l_blacklist") + bl = database.dbLoad("l_blacklist") return self.hostname in bl def penalties(self): @@ -449,15 +638,53 @@ class NodeRecord: def _get_contacts_for_condition(self): pass +class Action(MonRecord): + def __init__(self, host, data): + self.host = host + MonRecord.__init__(self, data) + return + + def deltaDays(self, delta): + t = datetime.fromtimestamp(self.__dict__['time']) + d = t + timedelta(delta) + self.__dict__['time'] = time.mktime(d.timetuple()) + +def node_end_record(node): + act_all = database.dbLoad("act_all") + if node not in act_all: + del act_all + return False + + if len(act_all[node]) == 0: + del act_all + return False + + a = Action(node, act_all[node][0]) + a.delField('rt') + a.delField('found_rt_ticket') + a.delField('second-mail-at-oneweek') + a.delField('second-mail-at-twoweeks') + a.delField('first-found') + rec = a.get() + rec['action'] = ["close_rt"] + rec['category'] = "UNKNOWN" + rec['stage'] = "monitor-end-record" + rec['time'] = time.time() - 7*60*60*24 + act_all[node].insert(0,rec) + database.dbDump("act_all", act_all) + del act_all + return True + if __name__ == "__main__": #r = RT() - #r.email("test", "body of test message", ['soltesz@cs.princeton.edu']) - from emailTxt import mailtxt - soltesz.dbDump("persistmessages", {}); - args = {'url_list': 'http://www.planet-lab.org/bootcds/planet1.usb\n','hostname': 'planet1','hostname_list': ' blahblah - days down\n'} - m = PersistMessage("blue", "test 1", mailtxt.newdown_one[1] % args, True) - m.send(['soltesz@cs.utk.edu']) - m = PersistMessage("blue", "test 1 - part 2", mailtxt.newalphacd_one[1] % args, True) + #r.email("test", "body of test message", ['database@cs.princeton.edu']) + #from emailTxt import mailtxt + print "loaded" + #database.dbDump("persistmessages", {}); + #args = {'url_list': 'http://www.planet-lab.org/bootcds/planet1.usb\n','hostname': 'planet1','hostname_list': ' blahblah - days down\n'} + #m = PersistMessage("blue", "test 1", mailtxt.newdown_one[1] % args, True) + #m.send(['soltesz@cs.utk.edu']) + #m = PersistMessage("blue", "test 1 - part 2", mailtxt.newalphacd_one[1] % args, True) # TRICK timer to thinking some time has passed. - m.actiontracker.time = time.time() - 6*60*60*24 - m.send(['soltesz@cs.utk.edu']) + #m.actiontracker.time = time.time() - 6*60*60*24 + #m.send(['soltesz@cs.utk.edu'])