#!/usr/bin/python
-import database
+from monitor import database
-import plc
-api = plc.getAuthAPI()
-
-import mailer
+from monitor.wrapper import plc, plccache
+from monitor.wrapper import mailer
import time
-from nodecommon import *
-from const import *
+from model import *
+from monitor.const import *
+from monitor import util
+from monitor import config
def gethostlist(hostlist_file):
- import config
- return config.getListFromFile(hostlist_file)
-
- #nodes = api.GetNodes({'peer_id' : None}, ['hostname'])
- #return [ n['hostname'] for n in nodes ]
+ return util.file.getListFromFile(hostlist_file)
def array_to_priority_map(array):
""" Create a mapping where each entry of array is given a priority equal
raise Exception("No index %s or %s in map" % (v1, v2))
def cmpCategoryVal(v1, v2):
+ # Terrible hack to manage migration to no more 'ALPHA' states.
+ if v1 == 'ALPHA': v1 = "PROD"
+ if v2 == 'ALPHA': v2 = "PROD"
+ #map = array_to_priority_map([ None, 'PROD', 'ALPHA', 'OLDBOOTCD', 'UNKNOWN', 'FORCED', 'ERROR', ])
map = array_to_priority_map([ None, 'ALPHA', 'PROD', 'OLDBOOTCD', 'UNKNOWN', 'FORCED', 'ERROR', ])
return cmpValMap(v1,v2,map)
# condition/penalty is applied, move to the next phase.
-fb = database.dbLoad("findbad")
-
class RT(object):
def __init__(self, ticket_id = None):
self.ticket_id = ticket_id
return self.status
def closeTicket(self):
- mailer.closeTicketViaRT(self.ticket_id)
+ mailer.closeTicketViaRT(self.ticket_id, "Ticket CLOSED automatically by SiteAssist.")
def email(self, subject, body, to):
self.ticket_id = mailer.emailViaRT(subject, body, to, self.ticket_id)
#print pm
if id in pm:
- print "Using existing object"
+ #print "Using existing object"
obj = pm[id]
else:
- print "creating new object"
+ #print "creating new object"
obj = super(PersistMessage, typ).__new__(typ, [id, subject, message, via_rt], **kwargs)
obj.id = id
- obj.actiontracker = Recent(3*60*60*24)
+ obj.actiontracker = Recent(1*60*60*24)
obj.ticket_id = None
if 'ticket_id' in kwargs and kwargs['ticket_id'] is not None:
def reset(self):
self.actiontracker.unsetRecent()
+ def save(self):
+ pm = database.dbLoad(self.db)
+ pm[self.id] = self
+ database.dbDump(self.db, pm)
+
def send(self, to):
if not self.actiontracker.isRecent():
self.ticket_id = Message.send(self, to)
self.actiontracker.setRecent()
-
- #print "recording object for persistance"
- pm = database.dbLoad(self.db)
- pm[self.id] = self
- database.dbDump(self.db, pm)
+ self.save()
else:
# NOTE: only send a new message every week, regardless.
- print "Not sending to host b/c not within window of %s days" % (self.actiontracker.withintime // 60*60*24)
+ # NOTE: can cause thank-you messages to be lost, for instance when node comes back online within window.
+ print "Not sending to host b/c not within window of %s days" % (self.actiontracker.withintime // (60*60*24))
class MonitorMessage(object):
def __new__(typ, id, *args, **kwargs):
def __init__(self, hostname, data):
self.hostname = hostname
self.data = data
- self.plcdb_hn2lb = database.dbLoad("plcdb_hn2lb")
+ self.plcdb_hn2lb = plccache.plcdb_hn2lb
self.loginbase = self.plcdb_hn2lb[self.hostname]
return
def severity(self):
category = self.data['category']
prev_category = self.data['prev_category']
+ #print "SEVERITY: ", category, prev_category
val = cmpCategoryVal(category, prev_category)
return val
def getDaysDown(cls, diag_record):
daysdown = -1
- if diag_record['comonstats']['uptime'] != "null":
+ if diag_record['comonstats']['uptime'] != "null" and diag_record['comonstats']['uptime'] != "-1":
daysdown = - int(float(diag_record['comonstats']['uptime'])) // (60*60*24)
#elif diag_record['comonstats']['sshstatus'] != "null":
# daysdown = int(diag_record['comonstats']['sshstatus']) // (60*60*24)
return daysdown
getStrDaysDown = classmethod(getStrDaysDown)
- #def getStrDaysDown(cls, diag_record):
- # daysdown = cls.getDaysDown(diag_record)
- # if daysdown > 0:
- # return "%d days down"%daysdown
- # elif daysdown == -1:
- # return "Never online"
- # else:
- # return "%d days up"% -daysdown
- #getStrDaysDown = classmethod(getStrDaysDown)
-
- def takeAction(self):
+ def getSendEmailFlag(self):
+ if not config.mail:
+ return False
+
+ # resend if open & created longer than 30 days ago.
+ if 'rt' in self.data and \
+ 'Status' in self.data['rt'] and \
+ "open" in self.data['rt']['Status'] and \
+ self.data['rt']['Created'] > int(time.time() - 60*60*24*30):
+ # if created-time is greater than the thirty days ago from the current time
+ return False
+
+ return True
+
+ def getMostRecentStage(self):
+ lastact = self.data['last_action_record']
+ return lastact.stage
+
+ def getMostRecentTime(self):
+ lastact = self.data['last_action_record']
+ return lastact.date_action_taken
+
+ def takeAction(self, index=0):
pp = PersistSitePenalty(self.hostname, 0, db='persistpenalty_hostnames')
- if 'improvement' in self.data['stage'] or self.improved():
- print "decreasing penalty for %s"%self.hostname
+ if 'improvement' in self.data['stage'] or self.improved() or \
+ 'monitor-end-record' in self.data['stage']:
+ print "takeAction: decreasing penalty for %s"%self.hostname
+ pp.decrease()
pp.decrease()
else:
- print "increasing penalty for %s"%self.hostname
+ print "takeAction: increasing penalty for %s"%self.hostname
pp.increase()
+ pp.index = index
pp.apply(self.hostname)
pp.save()
def _format_diaginfo(self):
info = self.data['info']
+ print "FORMAT : STAGE: ", self.data['stage']
if self.data['stage'] == 'monitor-end-record':
+ if info[2] == "ALPHA": info = (info[0], info[1], "PROD")
hlist = " %s went from '%s' to '%s'\n" % (info[0], info[1], info[2])
else:
hlist = " %s %s - %s\n" % (info[0], info[2], info[1]) #(node,ver,daysdn)
return hlist
+ def saveAction(self):
+ if 'save_act_all' in self.data and self.data['save_act_all'] == True:
+ return True
+ else:
+ return False
def getMessage(self, ticket_id=None):
self.data['args']['hostname'] = self.hostname
self.data['args']['loginbase'] = self.loginbase
self.data['args']['hostname_list'] = self._format_diaginfo()
- message = PersistMessage(self.hostname,
+ #print self.data['message']
+ if self.data['message']:
+ message = PersistMessage(self.hostname,
self.data['message'][0] % self.data['args'],
self.data['message'][1] % self.data['args'],
True, db='monitor_persistmessages',
ticket_id=ticket_id)
- return message
+ if self.data['stage'] == "improvement":
+ message.reset()
+ return message
+ else:
+ return None
def getContacts(self):
- from config import config
- #print "policy"
- config = config()
-
roles = self.data['email']
if not config.mail and not config.debug and config.bcc:
if ADMIN & roles:
contacts += [config.email]
if TECH & roles:
- contacts += [TECHEMAIL % self.loginbase]
+ #contacts += [TECHEMAIL % self.loginbase]
+ contacts += plc.getTechEmails(self.loginbase)
if PI & roles:
- contacts += [PIEMAIL % self.loginbase]
+ #contacts += [PIEMAIL % self.loginbase]
+ contacts += plc.getSliceUserEmails(self.loginbase)
if USER & roles:
+ contacts += plc.getSliceUserEmails(self.loginbase)
slices = plc.slices(self.loginbase)
if len(slices) >= 1:
- for slice in slices:
- contacts += [SLICEMAIL % slice]
+ #for slice in slices:
+ # contacts += [SLICEMAIL % slice]
print "SLIC: %20s : %d slices" % (self.loginbase, len(slices))
else:
print "SLIC: %20s : 0 slices" % self.loginbase
self.hostname = hostname
self.ticket = None
self.target = target
- if hostname in fb['nodes']:
- self.data = fb['nodes'][hostname]['values']
- else:
- raise Exception("Hostname not in scan database")
- def stageIswaitforever(self):
- if 'waitforever' in self.data['stage']:
- return True
- else:
- return False
-
- def severity(self):
- category = self.data['category']
- prev_category = self.data['prev_category']
- val = cmpCategoryVal(category, prev_category)
- return val
-
- def improved(self):
- return self.severity() > 0
-
- def end_record(self):
- return node_end_record(self.hostname)
-
- def reset_stage(self):
- self.data['stage'] = 'findbad'
- return True
-
- def open_tickets(self):
- if self.ticket and self.ticket.status['status'] == 'open':
- return 1
- return 0
- def setIntrospect(self):
- pass
-
- def email_notice(self):
- message = self._get_message_for_condition()
- message.send(self._get_contacts_for_condition())
- return True
- def close_ticket(self):
- if self.ticket:
- self.ticket.closeTicket()
-
- def exempt_from_penalties(self):
- bl = database.dbLoad("l_blacklist")
- return self.hostname in bl
-
- def penalties(self):
- return []
- def escellate_penalty(self):
- return True
- def reduce_penalty(self):
- return True
-
-
- def atTarget(self):
- return self.target.verify(self.data)
-
- def _get_condition(self):
- return self.data['category'].lower()
-
- def _get_stage(self):
- "improvement"
- "firstnotice_noop"
- "secondnotice_noslicecreation"
- "thirdnotice_disableslices"
-
- delta = current_time - self.data['time']
+class Action(MonRecord):
+ def __init__(self, host, data):
+ self.host = host
+ MonRecord.__init__(self, data)
+ return
- def _get_message_for_condition(self):
- pass
- def _get_contacts_for_condition(self):
- pass
+ def deltaDays(self, delta):
+ t = datetime.fromtimestamp(self.__dict__['time'])
+ d = t + timedelta(delta)
+ self.__dict__['time'] = time.mktime(d.timetuple())
+
+def node_end_record(node):
+ act_all = database.dbLoad("act_all")
+ if node not in act_all:
+ del act_all
+ return False
+
+ if len(act_all[node]) == 0:
+ del act_all
+ return False
+
+ pm = database.dbLoad("monitor_persistmessages")
+ if node not in pm:
+ del pm
+ return False
+ else:
+ print "deleting node record"
+ del pm[node]
+ database.dbDump("monitor_persistmessages", pm)
+
+ a = Action(node, act_all[node][0])
+ a.delField('rt')
+ a.delField('found_rt_ticket')
+ a.delField('second-mail-at-oneweek')
+ a.delField('second-mail-at-twoweeks')
+ a.delField('first-found')
+ rec = a.get()
+ rec['action'] = ["close_rt"]
+ rec['category'] = "ALPHA" # assume that it's up...
+ rec['stage'] = "monitor-end-record"
+ rec['ticket_id'] = None
+ rec['time'] = time.time() - 7*60*60*24
+ act_all[node].insert(0,rec)
+ database.dbDump("act_all", act_all)
+ del act_all
+ return True
if __name__ == "__main__":
#r = RT()
git://git.onelab.eu / monitor.git / blobdiff