+#!/usr/bin/python
#
# Copyright (c) 2004 The Trustees of Princeton University (Trustees).
#
# Faiyaz Ahmed <faiyaza@cs.princeton.edu>
#
-# $Id: $
+# $Id: monitor.py,v 1.4 2006/11/14 19:20:13 faiyaza Exp $
import sys
import os
import time
import logging
import Queue
+# Global config options
+import config
# daemonize and *pid
from util.process import *
# Email
import mailer
import emailTxt
-# Defaults
-debug = False
# Log to what
LOG="./monitor.log"
+# DAT
+DAT="./monitor.dat"
+
# Email defaults
MTA="localhost"
FROM="support@planet-lab.org"
+TECHEMAIL="tech-%s@sites.planet-lab.org"
+PIEMAIL="pi-%s@sites.planet-lab.org"
+
# API
XMLRPC_SERVER = 'https://www.planet-lab.org/PLCAPI/'
+
# Time between comon refresh
COSLEEP=300 #5mins
# Time to refresh DB and remove unused entries
for thread in runningthreads.keys():
# If thread found dead, remove from queue
if not runningthreads[thread].isAlive():
- logger.error("Thread Died: %s" %(thread))
+ logger.error("***********Thread died: %s**********" %(thread))
del runningthreads[thread]
"""
def main():
# Defaults
- global debug, status, logger
+ global status, logger
try:
longopts = ["debug", "status", "help"]
for (opt, optval) in opts:
if opt == "-d" or opt == "--debug":
- debug = True
+ config.debug = True
+ print "Running in DEBUG mode: NO EMAILS SENT AND NO SLICES SQUEEZED."
elif opt == "--status":
#print summary(names)
sys.exit(0)
startThread(ThreadWatcher(), "Watcher")
# The meat of it.
- # Nodes to check
- bucket = Queue.Queue()
+ # Nodes to check. Queue of all sick nodes.
+ toCheck = Queue.Queue()
+ # Nodes that are sick w/o tickets
+ sickNoTicket = Queue.Queue()
# Comon DB of all nodes
cdb = {}
# Nodes that are down. Use this to maintain DB; cleanup.
- alldown = Queue.Queue()
+ #alldown = Queue.Queue()
# RT DB
- tickets = {}
+ tickets = {}
+ # Nodes we've emailed.
+ # host - > (type of email, time)
+ emailed = {}
+
# Get RT Tickets.
- # Event based. Add to queue(bucket) and hosts are queried.
- rt1 = rt.RT(tickets, bucket)
- rt2 = rt.RT(tickets, bucket)
- rt3 = rt.RT(tickets, bucket)
- rt4 = rt.RT(tickets, bucket)
- rt5 = rt.RT(tickets, bucket)
+ # Event based. Add to queue(toCheck) and hosts are queried.
+ rt1 = rt.RT(tickets, toCheck, sickNoTicket)
+ rt2 = rt.RT(tickets, toCheck, sickNoTicket)
+ rt3 = rt.RT(tickets, toCheck, sickNoTicket)
+ rt4 = rt.RT(tickets, toCheck, sickNoTicket)
+ rt5 = rt.RT(tickets, toCheck, sickNoTicket)
# Kind of a hack. Cleans the DB for stale entries and updates db.
clean = Thread(target=rt5.cleanTickets)
# Poll Comon. Refreshes Comon data every COSLEEP seconds
- cm1 = comon.Comon(cdb, bucket)
+ cm1 = comon.Comon(cdb, toCheck)
+
+ # Actually digest the info and do something with it.
+ pol = policy.Policy(cm1, sickNoTicket, emailed)
+
+ # Load emailed sites from last run.
+ pol.emailedStore("LOAD")
# Start Threads
startThread(rt1,"rt1")
startThread(rt4,"rt4")
startThread(rt5,"rt5")
startThread(clean,"cleanrt5")
- startThread(cm1,"rt5")
- time.sleep(10)
- # Actually digest the info and do something with it.
- pol = policy.Policy(cm1, tickets)
+ # Start Comon Thread
+ startThread(cm1,"comon")
- while bucket.empty() == False:
- time.sleep(3)
+ # Wait for threads to init. Probably should join, but work on that later.
+ time.sleep(15)
+# while toCheck.empty() == false:
+# time.sleep(10)
+#
+ # Start Sending Emails
+ time.sleep(30)
startThread(pol, "policy")
- time.sleep(3600)
-
- #print runningthreads["RT"].ssh
+ time.sleep(10)
+
+ # Store state of emails
+# pol.emailedStore("WRITE")
+
+ # Email what we did.
+# pol.status()
logger.info('Monitor Exitted')
#if not debug: