import logging
import Queue
import time
+import re
import comon
+import soltesz
from threading import *
+# TODO: merge the RT mailer from mailer.py into this file.
+
# RT database access constants file
-RT_DB_CONSTANTS_PATH='/etc/planetlab/rt_db'
+RT_DB_CONSTANTS_PATH='rt_db'
#Logging
logger = logging.getLogger("monitor")
user=rt_db_constants['RT_DB_USER'],
passwd=rt_db_constants['RT_DB_PASSWORD'],
db=rt_db_constants['RT_DB_NAME'])
- except Error:
- print "Failed to connect to RT database"
+ except Exception, err:
+ print "Failed to connect to RT database: %s" %err
return -1
return rt_db
-def rt_tickets(hostname):
+def rt_tickets():
db = open_rt_db()
+ if db == -1:
+ return ""
# sql = """SELECT distinct Tk.id, Tk.Status, Tk.Subject
# FROM Tickets AS Tk
# JOIN Transactions AS Tr ON Tk.id=Tr.ObjectId
# Tk.Queue = 3 OR Tk.Queue = 19
# ORDER BY Tk.Status, Tk.LastUpdated DESC""" \
# % (hostname,hostname)
- sql = """SELECT distinct Tk.id, Tk.Status, Tk.Subject
- FROM Tickets AS Tk
- JOIN Transactions AS Tr ON Tk.id=Tr.ObjectId
- JOIN Attachments AS At ON Tr.id=At.TransactionID
- WHERE (At.Content LIKE '%%%s%%' OR
- At.Subject LIKE '%%%s%%') AND
- (Tk.Status = 'new' OR Tk.Status = 'open')
- ORDER BY Tk.Status, Tk.LastUpdated DESC""" \
- % (hostname,hostname)
+# sql = """SELECT distinct Tk.id, Tk.Status, Tk.Subject
+# FROM Tickets AS Tk
+# JOIN Transactions AS Tr ON Tk.id=Tr.ObjectId
+# JOIN Attachments AS At ON Tr.id=At.TransactionID
+# WHERE (At.Content LIKE '%%%s%%' OR
+# At.Subject LIKE '%%%s%%') AND
+# (Tk.Status = 'new' OR Tk.Status = 'open')
+# ORDER BY Tk.Status, Tk.LastUpdated DESC""" \
+# % (hostname,hostname)
+
+ # Queue == 10 is the spam Queue in RT.
+ sql = """SELECT distinct Tk.id, Tk.Status, Tk.Subject, At.Content
+ FROM Tickets AS Tk, Attachments AS At
+ JOIN Transactions AS Tr ON Tk.id=Tr.ObjectId
+ WHERE Tk.Queue != 10 AND Tk.id > 10000 AND
+ Tr.id=At.TransactionID AND Tk.Status = 'open'"""
+ #Tr.id=At.TransactionID AND (Tk.Status = 'new' OR Tk.Status = 'open')"""
try:
# create a 'cursor' (required by MySQLdb)
# map list of lists (raw) to list of dicts (tickets)
# when int gets pulls from SQL into python ints are converted to LONG to
# prevent overflow .. convert back
- tickets = map(lambda x: {"ticket_id":int(x[0]),
+ #tickets = map(lambda x: {"ticket_id":int(x[0]),
+ tickets = map(lambda x: {"ticket_id":str(x[0]),
"status":x[1],
- "subj":x[2]},
+ "subj":str(x[2]),
+ "content":str(x[3])},
raw)
db.close()
return tickets
+def is_host_in_rt_tickets(host, ticket_blacklist, ad_rt_tickets):
+ # ad_rt_tickets is an array of dicts, defined above.
+ if len(ad_rt_tickets) == 0:
+ return (False, None)
+
+ d_ticket = ad_rt_tickets[0]
+ if not ('ticket_id' in d_ticket and 'status' in d_ticket and
+ 'subj' in d_ticket and 'content' in d_ticket):
+ logger.debug("RT_tickets array has wrong fields!!!")
+ return (False, None)
+
+ #logger.debug("Searching all tickets for %s" % host)
+ def search_tickets(host, ad_rt_tickets):
+ # compile once for more efficiency
+ re_host = re.compile(host)
+ for x in ad_rt_tickets:
+ if re_host.search(x['subj'], re.MULTILINE|re.IGNORECASE) or \
+ re_host.search(x['content'], re.MULTILINE|re.IGNORECASE):
+ logger.debug("\t ticket %s has %s" % (x['ticket_id'], host))
+ print "\t ticket %s has %s" % (x['ticket_id'], host)
+ if x['ticket_id'] in ticket_blacklist:
+ return (False, x)
+ else:
+ return (True, x)
+ print "\t noticket -- has %s" % host
+ #logger.debug("\t noticket -- has %s" % host)
+ return (False, None)
+
+ # This search, while O(tickets), takes less than a millisecond, 05-25-07
+ #t = soltesz.MyTimer()
+ ret = search_tickets(host, ad_rt_tickets)
+ #del t
+
+ return ret
+
'''
Finds tickets associated with hostnames.
Another thread refresh tickets of nodes already in dict and remove nodes that have come up.
'''
class RT(Thread):
- def __init__(self, tickets, toCheck, sickNoTicket, target = None):
+ def __init__(self, dbTickets, q_toRT, q_fromRT, l_ticket_blacklist, target = None):
# Time of last update of ticket DB
+ self.dbTickets = dbTickets
self.lastupdated = 0
- # Queue() is MP/MC self locking.
- # Check host in queue. Queue populated from comon data of sick.
- self.toCheck = toCheck
- # Result of rt db query. Nodes without tickets that are sick.
- self.sickNoTicket = sickNoTicket
- #DB of tickets. Name -> ticket
- self.tickets = tickets
+ self.l_ticket_blacklist = l_ticket_blacklist
+ self.q_toRT = q_toRT
+ self.q_fromRT = q_fromRT
+ self.tickets = {}
Thread.__init__(self,target = self.getTickets)
- # Takes node from toCheck, gets tickets.
+ # Takes node from q_toRT, gets tickets.
# Thread that actually gets the tickets.
def getTickets(self):
+ self.count = 0
while 1:
- host = self.toCheck.get(block = True)
- if host == "None": break
- #if self.tickets.has_key(host) == False:
- #logger.debug("Popping from q - %s" %host)
- tmp = rt_tickets(host)
- if tmp:
- #logger.debug("RT: tickets for %s" %host)
- self.tickets[host] = tmp
+ diag_node = self.q_toRT.get(block = True)
+ if diag_node != None:
+ host = diag_node['nodename']
+ (b_host_inticket, r_ticket) = is_host_in_rt_tickets(host, \
+ self.l_ticket_blacklist, \
+ self.dbTickets)
+ diag_node['found_rt_ticket'] = None
+ if b_host_inticket:
+ logger.debug("RT: found tickets for %s" %host)
+ diag_node['found_rt_ticket'] = r_ticket['ticket_id']
+
+ else:
+ if r_ticket is not None:
+ print "Ignoring ticket %s" % r_ticket['ticket_id']
+ # TODO: why do i return the ticket id for a
+ # blacklisted ticket id?
+ #diag_node['found_rt_ticket'] = r_ticket['ticket_id']
+ self.count = self.count + 1
+
+ self.q_fromRT.put(diag_node)
else:
- logger.debug("RT: no tix for %s - policy" %host)
- self.sickNoTicket.put(host)
+ print "RT processed %d nodes with noticket" % self.count
+ logger.debug("RT filtered %d noticket nodes" % self.count)
+ self.q_fromRT.put(None)
+
+ break
# Removes hosts that are no longer down.
def remTickets(self):
prevdown = self.tickets.keys()
currdown = []
- #BEGIN HACK. This should be outside of this file. passed to class.
- cmn = comon.Comon(None, None)
- cmn.updatebkts()
- for bucket in cmn.comonbkts.keys():
- for host in getattr(cmn,bucket):
- if host not in currdown: currdown.append(host)
- #END HACK
+ ##BEGIN HACK. This should be outside of this file. passed to class.
+ #cmn = comon.Comon(None, None)
+ # cmn.updatebkts()
+ #for bucket in cmn.comonbkts.keys():
+ # for host in getattr(cmn,bucket):
+ # if host not in currdown: currdown.append(host)
+ ##END HACK
# Actually do the comparison
- for host in prevdown:
- if host not in currdown:
- del self.tickets[host]
- logger.info("RT: %s no longer down." % host)
+ #for host in prevdown:
+ # if host not in currdown:
+ # del self.tickets[host]
+ # logger.info("RT: %s no longer down." % host)
# Update Tickets
def updateTickets(self):
logger.info("Refreshing DB.")
for host in self.tickets.keys():
# Put back in Q to refresh
- self.toCheck.put(host)
+ self.q_toRT.put(host)
def cleanTickets(self):
while 1:
git://git.onelab.eu / monitor.git / blobdiff