-class Action(Thread):
- def __init__(self, l_action):
- self.l_action = l_action
-
- # the hostname to loginbase mapping
- self.plcdb_hn2lb = soltesz.dbLoad("plcdb_hn2lb")
-
- # Actions to take.
- self.diagnose_db = soltesz.if_cached_else(1, "diagnose_out", lambda : {})
- # Actions taken.
- self.act_all = soltesz.if_cached_else(1, "act_all", lambda : {})
-
- # A dict of actions to specific functions. PICKLE doesnt' like lambdas.
- self.actions = {}
- self.actions['suspendslices'] = lambda args: plc.suspendSlices(args['hostname'])
- self.actions['nocreate'] = lambda args: plc.removeSliceCreation(args['hostname'])
- self.actions['close_rt'] = lambda args: close_rt_backoff(args)
- self.actions['rins'] = lambda args: plc.nodeBootState(args['hostname'], "rins")
- self.actions['noop'] = lambda args: args
- self.actions['reset_nodemanager'] = lambda args: args # reset_nodemanager(args)
-
- self.actions['ticket_waitforever'] = lambda args: args
- self.actions['waitforever'] = lambda args: args
- self.actions['unknown'] = lambda args: args
- self.actions['waitforoneweekaction'] = lambda args: args
- self.actions['waitfortwoweeksaction'] = lambda args: args
- self.actions['sendmailagain-waitforoneweekaction'] = lambda args: args
- self.actions['sendmailagain-waitfortwoweeksaction'] = lambda args: args
- self.actions['email-againwaitforever'] = lambda args: args
- self.actions['email-againticket_waitforever'] = lambda args: args
-
-
- self.sickdb = {}
- Thread.__init__(self)
-
- def run(self):
- self.accumSites()
- print "Accumulated %d sick sites" % len(self.sickdb.keys())
- logger.debug("Accumulated %d sick sites" % len(self.sickdb.keys()))
-
- try:
- stats = self.analyseSites()
- except Exception, err:
- print "----------------"
- import traceback
- print traceback.print_exc()
- print err
- if config.policysavedb:
- print "Saving Databases... act_all"
- soltesz.dbDump("act_all", self.act_all)
- sys.exit(1)
-
- print_stats("sites_observed", stats)
- print_stats("sites_diagnosed", stats)
- print_stats("nodes_diagnosed", stats)
- print_stats("sites_emailed", stats)
- print_stats("nodes_actedon", stats)
- print string.join(stats['allsites'], ",")
-
- if config.policysavedb:
- print "Saving Databases... act_all"
- #soltesz.dbDump("policy.eventlog", self.eventlog)
- # TODO: remove 'diagnose_out',
- # or at least the entries that were acted on.
- soltesz.dbDump("act_all", self.act_all)
-
- def accumSites(self):
- """
- Take all nodes, from l_action, look them up in the diagnose_db database,
- and insert them into sickdb[] as:
-
- This way only the given l_action nodes will be acted on regardless
- of how many from diagnose_db are available.
-
- sickdb[loginbase][nodename] = diag_record
- """
- # TODO: what if l_action == None ?
- for nodename in self.l_action:
-
- loginbase = self.plcdb_hn2lb[nodename]
-
- if loginbase in self.diagnose_db and \
- nodename in self.diagnose_db[loginbase]['nodes']:
-
- diag_record = self.diagnose_db[loginbase]['nodes'][nodename]
-
- if loginbase not in self.sickdb:
- self.sickdb[loginbase] = {'nodes' : {}}
-
- # NOTE: don't copy all node records, since not all will be in l_action
- self.sickdb[loginbase]['nodes'][nodename] = diag_record
- # NOTE: but, we want to get the loginbase config settings,
- # this is the easiest way.
- self.sickdb[loginbase]['config'] = self.diagnose_db[loginbase]['config']
- #else:
- #print "%s not in diagnose_db!!" % loginbase
- return
-
- def __emailSite(self, loginbase, roles, message, args):
- """
- loginbase is the unique site abbreviation, prepended to slice names.
- roles contains TECH, PI, USER roles, and derive email aliases.
- record contains {'message': [<subj>,<body>], 'args': {...}}
- """
- ticket_id = 0
- args.update({'loginbase':loginbase})
-
- if not config.mail and not config.debug and config.bcc:
- roles = ADMIN
- if config.mail and config.debug:
- roles = ADMIN
-
- # build targets
- contacts = []
- if ADMIN & roles:
- contacts += [config.email]
- if TECH & roles:
- contacts += [TECHEMAIL % loginbase]
- if PI & roles:
- contacts += [PIEMAIL % loginbase]
- if USER & roles:
- slices = plc.slices(loginbase)
- if len(slices) >= 1:
- for slice in slices:
- contacts += [SLICEMAIL % slice]
- print "SLIC: %20s : %d slices" % (loginbase, len(slices))
- else:
- print "SLIC: %20s : 0 slices" % loginbase
-
- try:
- subject = message[0] % args
- body = message[1] % args
- if ADMIN & roles:
- # send only to admin
- if 'ticket_id' in args:
- subj = "Re: [PL #%s] %s" % (args['ticket_id'], subject)
- else:
- subj = "Re: [PL noticket] %s" % subject
- mailer.email(subj, body, contacts)
- ticket_id = args['ticket_id']
- else:
- ticket_id = mailer.emailViaRT(subject, body, contacts, args['ticket_id'])
- except Exception, err:
- print "exception on message:"
- import traceback
- print traceback.print_exc()
- print message
-
- return ticket_id
-
-
- def _format_diaginfo(self, diag_node):
- info = diag_node['info']
- if diag_node['stage'] == 'monitor-end-record':
- hlist = " %s went from '%s' to '%s'\n" % (info[0], info[1], info[2])
- else:
- hlist = " %s %s - %s\n" % (info[0], info[2], info[1]) #(node,ver,daysdn)
- return hlist
-
-
- def get_email_args(self, act_recordlist):
-
- email_args = {}
- email_args['hostname_list'] = ""
-
- for act_record in act_recordlist:
- email_args['hostname_list'] += act_record['msg_format']
- email_args['hostname'] = act_record['nodename']
- if 'ticket_id' in act_record:
- email_args['ticket_id'] = act_record['ticket_id']
-
- return email_args
-
- def get_unique_issues(self, act_recordlist):
- # NOTE: only send one email per site, per problem...
- unique_issues = {}
- for act_record in act_recordlist:
- act_key = act_record['action'][0]
- if act_key not in unique_issues:
- unique_issues[act_key] = []
-
- unique_issues[act_key] += [act_record]
-
- return unique_issues
-
-
- def __actOnSite(self, loginbase, site_record):
- i_nodes_actedon = 0
- i_nodes_emailed = 0
-
- act_recordlist = []
-
- for nodename in site_record['nodes'].keys():
- diag_record = site_record['nodes'][nodename]
- act_record = self.__actOnNode(diag_record)
- #print "nodename: %s %s" % (nodename, act_record)
- act_recordlist += [act_record]
-
- unique_issues = self.get_unique_issues(act_recordlist)
-
- for issue in unique_issues.keys():
- print "\tworking on issue: %s" % issue
- issue_record_list = unique_issues[issue]
- email_args = self.get_email_args(issue_record_list)
-
- act_record = issue_record_list[0]
- # send message before squeezing
- print "\t\tconfig.email: %s and %s" % (act_record['message'] != None,
- site_record['config']['email'])
- if act_record['message'] != None and site_record['config']['email']:
- ticket_id = self.__emailSite(loginbase, act_record['email'],
- act_record['message'], email_args)
-
- # Add ticket_id to ALL nodenames
- for act_record in issue_record_list:
- nodename = act_record['nodename']
- # update node record with RT ticket_id
- if nodename in self.act_all:
- self.act_all[nodename][0]['ticket_id'] = "%s" % ticket_id
- if config.mail: i_nodes_emailed += 1
-
- print "\t\tconfig.squeeze: %s and %s" % (config.squeeze,
- site_record['config']['squeeze'])
- if config.squeeze and site_record['config']['squeeze']:
- for act_key in act_record['action']:
- self.actions[act_key](email_args)
- i_nodes_actedon += 1
-
- if config.policysavedb:
- print "Saving Databases... act_all, diagnose_out"
- soltesz.dbDump("act_all", self.act_all)
- # remove site record from diagnose_out, it's in act_all as done.
- del self.diagnose_db[loginbase]
- soltesz.dbDump("diagnose_out", self.diagnose_db)
-
- print "sleeping for 1 sec"
- time.sleep(1)
- #print "Hit enter to continue..."
- #sys.stdout.flush()
- #line = sys.stdin.readline()
-
- return (i_nodes_actedon, i_nodes_emailed)
-
- def __actOnNode(self, diag_record):
- nodename = diag_record['nodename']
- message = diag_record['message']
-
- act_record = {}
- act_record.update(diag_record)
- act_record['nodename'] = nodename
- act_record['msg_format'] = self._format_diaginfo(diag_record)
-
- print "%s" % act_record['log'],
- print "%15s" % act_record['action']
-
- if act_record['stage'] is not 'monitor-end-record' and \
- act_record['stage'] is not 'nmreset':
- if nodename not in self.act_all:
- self.act_all[nodename] = []
-
- self.act_all[nodename].insert(0,act_record)
- else:
- print "Not recording %s in act_all" % nodename
-
- return act_record
-
- def analyseSites(self):
- i_sites_observed = 0
- i_sites_diagnosed = 0
- i_nodes_diagnosed = 0
- i_nodes_actedon = 0
- i_sites_emailed = 0
- l_allsites = []
-
- sorted_sites = self.sickdb.keys()
- sorted_sites.sort()
- for loginbase in sorted_sites:
- site_record = self.sickdb[loginbase]
- print "sites: %s" % loginbase
-
- i_nodes_diagnosed += len(site_record.keys())
- i_sites_diagnosed += 1
-
- (na,ne) = self.__actOnSite(loginbase, site_record)
-
- i_sites_observed += 1
- i_nodes_actedon += na
- i_sites_emailed += ne
-
- l_allsites += [loginbase]
-
- return {'sites_observed': i_sites_observed,
- 'sites_diagnosed': i_sites_diagnosed,
- 'nodes_diagnosed': i_nodes_diagnosed,
- 'sites_emailed': i_sites_emailed,
- 'nodes_actedon': i_nodes_actedon,
- 'allsites':l_allsites}
-
- def print_stats(self, key, stats):
- print "%20s : %d" % (key, stats[key])
-
-
-
- #"""
- #Prints, logs, and emails status of up nodes, down nodes, and buckets.
- #"""
- #def status(self):
- # sub = "Monitor Summary"
- # msg = "\nThe following nodes were acted upon: \n\n"
- # for (node, (type, date)) in self.emailed.items():
- # # Print only things acted on today.
- # if (time.gmtime(time.time())[2] == time.gmtime(date)[2]):
- # msg +="%s\t(%s)\t%s\n" %(node, type, time.ctime(date))
- # msg +="\n\nThe following sites have been 'squeezed':\n\n"
- # for (loginbase, (date, type)) in self.squeezed.items():
- # # Print only things acted on today.
- # if (time.gmtime(time.time())[2] == time.gmtime(date)[2]):
- # msg +="%s\t(%s)\t%s\n" %(loginbase, type, time.ctime(date))
- # mailer.email(sub, msg, [SUMTO])
- # logger.info(msg)
- # return
-
- #"""
- #Store/Load state of emails. When, where, what.
- #"""
- #def emailedStore(self, action):
- # try:
- # if action == "LOAD":
- # f = open(DAT, "r+")
- # logger.info("POLICY: Found and reading " + DAT)
- # self.emailed.update(pickle.load(f))
- # if action == "WRITE":
- # f = open(DAT, "w")
- # #logger.debug("Writing " + DAT)
- # pickle.dump(self.emailed, f)
- # f.close()
- # except Exception, err:
- # logger.info("POLICY: Problem with DAT, %s" %err)
-
-
-#class Policy(Thread):