add more info to sliceavg
[monitor.git] / monitor / model.py
1 #!/usr/bin/python
2
3 from monitor import database
4
5 from monitor.wrapper import plc
6 from monitor.wrapper import mailer
7 import time
8
9 from monitor.const import *
10 from monitor import util
11 from monitor import config
12
13 import time
14 from datetime import datetime, timedelta
15 import re
16
17 def gethostlist(hostlist_file):
18         return util.file.getListFromFile(hostlist_file)
19
20 def array_to_priority_map(array):
21         """ Create a mapping where each entry of array is given a priority equal
22         to its position in the array.  This is useful for subsequent use in the
23         cmpMap() function."""
24         map = {}
25         count = 0
26         for i in array:
27                 map[i] = count
28                 count += 1
29         return map
30
31 def cmpValMap(v1, v2, map):
32         if v1 in map and v2 in map and map[v1] < map[v2]:
33                 return 1
34         elif v1 in map and v2 in map and map[v1] > map[v2]:
35                 return -1
36         elif v1 in map and v2 in map:
37                 return 0
38         else:
39                 raise Exception("No index %s or %s in map" % (v1, v2))
40
41 def cmpCategoryVal(v1, v2):
42         # Terrible hack to manage migration to no more 'ALPHA' states.
43         if v1 == 'ALPHA': v1 = "PROD"
44         if v2 == 'ALPHA': v2 = "PROD"
45         #map = array_to_priority_map([ None, 'PROD', 'ALPHA', 'OLDBOOTCD', 'UNKNOWN', 'FORCED', 'ERROR', ])
46         map = array_to_priority_map([ None, 'ALPHA', 'PROD', 'OLDBOOTCD', 'UNKNOWN', 'FORCED', 'ERROR', ])
47         return cmpValMap(v1,v2,map)
48
49
50 class PCU:
51         def __init__(self, hostname):
52                 self.hostname = hostname
53
54         def reboot(self):
55                 return True
56         def available(self):
57                 return True
58         def previous_attempt(self):
59                 return True
60         def setValidMapping(self):
61                 pass
62
63 class Penalty:
64         def __init__(self, key, valuepattern, action):
65                 pass
66
67 class PenaltyMap:
68         def __init__(self):
69                 pass
70
71         # connect one penalty to another, in a FSM diagram.  After one
72         #       condition/penalty is applied, move to the next phase.
73
74
75 class RT(object):
76         def __init__(self, ticket_id = None):
77                 self.ticket_id = ticket_id
78                 if self.ticket_id:
79                         print "getting ticket status",
80                         self.status = mailer.getTicketStatus(self.ticket_id)
81                         print self.status
82
83         def setTicketStatus(self, status):
84                 mailer.setTicketStatus(self.ticket_id, status)
85                 self.status = mailer.getTicketStatus(self.ticket_id)
86                 return True
87         
88         def getTicketStatus(self):
89                 if not self.status:
90                         self.status = mailer.getTicketStatus(self.ticket_id)
91                 return self.status
92
93         def closeTicket(self):
94                 mailer.closeTicketViaRT(self.ticket_id, "Ticket CLOSED automatically by SiteAssist.") 
95
96         def email(self, subject, body, to):
97                 self.ticket_id = mailer.emailViaRT(subject, body, to, self.ticket_id)
98                 return self.ticket_id
99
100 class Message(object):
101         def __init__(self, subject, message, via_rt=True, ticket_id=None, **kwargs):
102                 self.via_rt = via_rt
103                 self.subject = subject
104                 self.message = message
105                 self.rt = RT(ticket_id)
106
107         def send(self, to):
108                 if self.via_rt:
109                         return self.rt.email(self.subject, self.message, to)
110                 else:
111                         return mailer.email(self.subject, self.message, to)
112
113 class Recent(object):
114         def __init__(self, withintime):
115                 self.withintime = withintime
116
117                 try:
118                         self.time = self.__getattribute__('time')
119                 except:
120                         self.time = time.time()- 7*24*60*60
121
122                 #self.time = time.time()
123                 #self.action_taken = False
124
125         def isRecent(self):
126                 if self.time + self.withintime < time.time():
127                         self.action_taken = False
128
129                 if self.time + self.withintime > time.time() and self.action_taken:
130                         return True
131                 else:
132                         return False
133
134         def unsetRecent(self):
135                 self.action_taken = False
136                 self.time = time.time()
137                 return True
138
139         def setRecent(self):
140                 self.action_taken = True
141                 self.time = time.time()
142                 return True
143                 
144 class PersistFlags(Recent):
145         def __new__(typ, id, *args, **kwargs):
146                 if 'db' in kwargs:
147                         db = kwargs['db']
148                         del kwargs['db']
149                 else:
150                         db = "persistflags"
151
152                 try:
153                         pm = database.dbLoad(db)
154                 except:
155                         database.dbDump(db, {})
156                         pm = database.dbLoad(db)
157                 #print pm
158                 if id in pm:
159                         obj = pm[id]
160                 else:
161                         obj = super(PersistFlags, typ).__new__(typ, *args, **kwargs)
162                         for key in kwargs.keys():
163                                 obj.__setattr__(key, kwargs[key])
164                         obj.time = time.time()
165                         obj.action_taken = False
166
167                 obj.db = db
168                 return obj
169
170         def __init__(self, id, withintime, **kwargs):
171                 self.id = id
172                 Recent.__init__(self, withintime)
173
174         def save(self):
175                 pm = database.dbLoad(self.db)
176                 pm[self.id] = self
177                 database.dbDump(self.db, pm)
178
179         def resetFlag(self, name):
180                 self.__setattr__(name, False)
181
182         def setFlag(self, name):
183                 self.__setattr__(name, True)
184                 
185         def getFlag(self, name):
186                 try:
187                         return self.__getattribute__(name)
188                 except:
189                         self.__setattr__(name, False)
190                         return False
191
192         def resetRecentFlag(self, name):
193                 self.resetFlag(name)
194                 self.unsetRecent()
195
196         def setRecentFlag(self, name):
197                 self.setFlag(name)
198                 self.setRecent()
199
200         def getRecentFlag(self, name):
201                 # if recent and flag set -> true
202                 # else false
203                 try:
204                         return self.isRecent() & self.__getattribute__(name)
205                 except:
206                         self.__setattr__(name, False)
207                         return False
208
209         def checkattr(self, name):
210                 try:
211                         x = self.__getattribute__(name)
212                         return True
213                 except:
214                         return False
215                 
216
217 class PersistMessage(Message):
218         def __new__(typ, id, subject, message, via_rt, **kwargs):
219                 if 'db' in kwargs:
220                         db = kwargs['db']
221                 else:
222                         db = "persistmessages"
223
224                 try:
225                         pm = database.dbLoad(db)
226                 except:
227                         database.dbDump(db, {})
228                         pm = database.dbLoad(db)
229
230                 #print pm
231                 if id in pm:
232                         #print "Using existing object"
233                         obj = pm[id]
234                 else:
235                         #print "creating new object"
236                         obj = super(PersistMessage, typ).__new__(typ, [id, subject, message, via_rt], **kwargs)
237                         obj.id = id
238                         obj.actiontracker = Recent(1*60*60*24)
239                         obj.ticket_id = None
240
241                 if 'ticket_id' in kwargs and kwargs['ticket_id'] is not None:
242                         obj.ticket_id = kwargs['ticket_id']
243
244                 obj.db = db
245                 return obj
246
247         def __init__(self, id, subject, message, via_rt=True, **kwargs):
248                 print "initializing object: %s" % self.ticket_id
249                 self.id = id
250                 Message.__init__(self, subject, message, via_rt, self.ticket_id)
251
252         def reset(self):
253                 self.actiontracker.unsetRecent()
254
255         def save(self):
256                 pm = database.dbLoad(self.db)
257                 pm[self.id] = self
258                 database.dbDump(self.db, pm)
259
260         def send(self, to):
261                 if not self.actiontracker.isRecent():
262                         self.ticket_id = Message.send(self, to)
263                         self.actiontracker.setRecent()
264                         self.save()
265                 else:
266                         # NOTE: only send a new message every week, regardless.
267                         # NOTE: can cause thank-you messages to be lost, for instance when node comes back online within window.
268                         print "Not sending to host b/c not within window of %s days" % (self.actiontracker.withintime // (60*60*24))
269
270 class MonitorMessage(object):
271         def __new__(typ, id, *args, **kwargs):
272                 if 'db' in kwargs:
273                         db = kwargs['db']
274                 else:
275                         db = "monitormessages"
276
277                 try:
278                         if 'reset' in kwargs and kwargs['reset'] == True:
279                                 database.dbDump(db, {})
280                         pm = database.dbLoad(db)
281                 except:
282                         database.dbDump(db, {})
283                         pm = database.dbLoad(db)
284
285                 #print pm
286                 if id in pm:
287                         print "Using existing object"
288                         obj = pm[id]
289                 else:
290                         print "creating new object"
291                         obj = super(object, typ).__new__(typ, id, *args, **kwargs)
292                         obj.id = id
293                         obj.sp = PersistSitePenalty(id, 0)
294
295                 obj.db = db
296                 return obj
297
298         def __init__(self, id, message):
299                 pass
300                 
301
302 class SitePenalty(object):
303         penalty_map = [] 
304         penalty_map.append( { 'name': 'noop',                   'enable'   : lambda host: None,
305                                                                                                         'disable'  : lambda host: None } )
306         penalty_map.append( { 'name': 'nocreate',               'enable'   : lambda host: plc.removeSliceCreation(host),
307                                                                                                         'disable'  : lambda host: plc.enableSliceCreation(host) } )
308         penalty_map.append( { 'name': 'suspendslices',  'enable'   : lambda host: plc.suspendSlices(host),
309                                                                                                         'disable'  : lambda host: plc.enableSlices(host) } )
310
311         #def __init__(self, index=0, **kwargs):
312         #       self.index = index
313
314         def get_penalties(self):
315                 # TODO: get penalties actually applied to a node from PLC DB.
316                 return [ n['name'] for n in SitePenalty.penalty_map ] 
317
318         def increase(self):
319                 self.index = self.index + 1
320                 if self.index > len(SitePenalty.penalty_map)-1: self.index = len(SitePenalty.penalty_map)-1
321                 return True
322
323         def decrease(self):
324                 self.index = self.index - 1
325                 if self.index < 0: self.index = 0
326                 return True
327
328         def apply(self, host):
329
330                 for i in range(len(SitePenalty.penalty_map)-1,self.index,-1):
331                         print "\tdisabling %s on %s" % (SitePenalty.penalty_map[i]['name'], host)
332                         SitePenalty.penalty_map[i]['disable'](host)
333
334                 for i in range(0,self.index+1):
335                         print "\tapplying %s on %s" % (SitePenalty.penalty_map[i]['name'], host)
336                         SitePenalty.penalty_map[i]['enable'](host)
337
338                 return
339
340
341
342 class PersistSitePenalty(SitePenalty):
343         def __new__(typ, id, index, **kwargs):
344                 if 'db' in kwargs:
345                         db = kwargs['db']
346                 else:
347                         db = "persistpenalties"
348
349                 try:
350                         if 'reset' in kwargs and kwargs['reset'] == True:
351                                 database.dbDump(db, {})
352                         pm = database.dbLoad(db)
353                 except:
354                         database.dbDump(db, {})
355                         pm = database.dbLoad(db)
356
357                 #print pm
358                 if id in pm:
359                         print "Using existing object"
360                         obj = pm[id]
361                 else:
362                         print "creating new object"
363                         obj = super(PersistSitePenalty, typ).__new__(typ, [index], **kwargs)
364                         obj.id = id
365                         obj.index = index
366
367                 obj.db = db
368                 return obj
369
370         def __init__(self, id, index, **kwargs):
371                 self.id = id
372
373         def save(self):
374                 pm = database.dbLoad(self.db)
375                 pm[self.id] = self
376                 database.dbDump(self.db, pm)
377
378
379 class Target:
380         """
381                 Each host has a target set of attributes.  Some may be set manually,
382                 or others are set globally for the preferred target.
383
384                 For instance:
385                         All nodes in the Alpha or Beta group would have constraints like:
386                                 [ { 'state' : 'BOOT', 'kernel' : '2.6.22' } ]
387         """
388         def __init__(self, constraints):
389                 self.constraints = constraints
390
391         def verify(self, data):
392                 """
393                         self.constraints is a list of key, value pairs.
394                         # [ {... : ...}==AND , ... , ... , ] == OR
395                 """
396                 con_or_true = False
397                 for con in self.constraints:
398                         #print "con: %s" % con
399                         con_and_true = True
400                         for key in con.keys():
401                                 #print "looking at key: %s" % key
402                                 if key in data: 
403                                         #print "%s %s" % (con[key], data[key])
404                                         con_and_true = con_and_true & (con[key] in data[key])
405                                 elif key not in data:
406                                         print "missing key %s" % key
407                                         con_and_true = False
408
409                         con_or_true = con_or_true | con_and_true
410
411                 return con_or_true
412
413 class Record(object):
414
415         def __init__(self, hostname, data):
416                 from monitor.wrapper import plccache
417                 self.hostname = hostname
418                 self.data = data
419                 self.plcdb_hn2lb = plccache.plcdb_hn2lb
420                 self.loginbase = self.plcdb_hn2lb[self.hostname]
421                 return
422
423
424         def stageIswaitforever(self):
425                 if 'waitforever' in self.data['stage']:
426                         return True
427                 else:
428                         return False
429
430         def severity(self):
431                 category = self.data['category']
432                 prev_category = self.data['prev_category']
433                 #print "SEVERITY: ", category, prev_category
434                 val = cmpCategoryVal(category, prev_category)
435                 return val 
436
437         def improved(self):
438                 return self.severity() > 0
439         
440         def end_record(self):
441                 return node_end_record(self.hostname)
442
443         def reset_stage(self):
444                 self.data['stage'] = 'findbad'
445                 return True
446         
447         def getCategory(self):
448                 return self.data['category'].lower()
449
450         def getState(self):
451                 return self.data['state'].lower()
452
453         def getDaysDown(cls, diag_record):
454                 daysdown = -1
455                 if diag_record['comonstats']['uptime'] != "null" and diag_record['comonstats']['uptime'] != "-1":
456                         daysdown = - int(float(diag_record['comonstats']['uptime'])) // (60*60*24)
457                 #elif diag_record['comonstats']['sshstatus'] != "null":
458                 #       daysdown = int(diag_record['comonstats']['sshstatus']) // (60*60*24)
459                 #elif diag_record['comonstats']['lastcotop'] != "null":
460                 #       daysdown = int(diag_record['comonstats']['lastcotop']) // (60*60*24)
461                 else:
462                         now = time.time()
463                         last_contact = diag_record['plcnode']['last_contact']
464                         if last_contact == None:
465                                 # the node has never been up, so give it a break
466                                 daysdown = -1
467                         else:
468                                 diff = now - last_contact
469                                 daysdown = diff // (60*60*24)
470                 return daysdown
471         getDaysDown = classmethod(getDaysDown)
472
473         def getStrDaysDown(cls, diag_record):
474                 daysdown = "unknown"
475                 last_contact = diag_record['plcnode']['last_contact']
476                 date_created = diag_record['plcnode']['date_created']
477
478                 if      diag_record['comonstats']['uptime'] != "null" and \
479                         diag_record['comonstats']['uptime'] != "-1":
480                         daysdown = int(float(diag_record['comonstats']['uptime'])) // (60*60*24)
481                         daysdown = "%d days up" % daysdown
482
483                 elif last_contact is None:
484                         if date_created is not None:
485                                 now = time.time()
486                                 diff = now - date_created
487                                 daysdown = diff // (60*60*24)
488                                 daysdown = "Never contacted PLC, created %s days ago" % daysdown
489                         else:
490                                 daysdown = "Never contacted PLC"
491                 else:
492                         now = time.time()
493                         diff = now - last_contact
494                         daysdown = diff // (60*60*24)
495                         daysdown = "%s days down" % daysdown
496                 return daysdown
497         getStrDaysDown = classmethod(getStrDaysDown)
498
499         def getSendEmailFlag(self):
500                 if not config.mail:
501                         return False
502
503                 # resend if open & created longer than 30 days ago.
504                 if  'rt' in self.data and \
505                         'Status' in self.data['rt'] and \
506                         "open" in self.data['rt']['Status'] and \
507                         self.data['rt']['Created'] > int(time.time() - 60*60*24*30):
508                         # if created-time is greater than the thirty days ago from the current time
509                         return False
510
511                 return True
512
513         def getMostRecentStage(self):
514                 lastact = self.data['last_action_record']
515                 return lastact.stage
516
517         def getMostRecentTime(self):
518                 lastact = self.data['last_action_record']
519                 return lastact.date_action_taken
520
521         def takeAction(self, index=0):
522                 pp = PersistSitePenalty(self.hostname, 0, db='persistpenalty_hostnames')
523                 if 'improvement' in self.data['stage'] or self.improved() or \
524                         'monitor-end-record' in self.data['stage']:
525                         print "takeAction: decreasing penalty for %s"%self.hostname
526                         pp.decrease()
527                         pp.decrease()
528                 else:
529                         print "takeAction: increasing penalty for %s"%self.hostname
530                         pp.increase()
531
532                 print "takeAction: applying penalty to %s as index %s"% (self.hostname, index)
533                 pp.index = index
534                 pp.apply(self.hostname)
535                 pp.save()
536
537         def _format_diaginfo(self):
538                 info = self.data['info']
539                 print "FORMAT : STAGE: ", self.data['stage']
540                 if self.data['stage'] == 'monitor-end-record':
541                         if info[2] == "ALPHA": info = (info[0], info[1], "PROD")
542                         hlist = "    %s went from '%s' to '%s'\n" % (info[0], info[1], info[2]) 
543                 else:
544                         hlist = "    %s %s - %s\n" % (info[0], info[2], info[1]) #(node,ver,daysdn)
545                 return hlist
546         def saveAction(self):
547                 if 'save_act_all' in self.data and self.data['save_act_all'] == True:
548                         return True
549                 else:
550                         return False
551
552         def getMessage(self, ticket_id=None):
553                 self.data['args']['hostname'] = self.hostname
554                 self.data['args']['loginbase'] = self.loginbase
555                 self.data['args']['hostname_list'] = self._format_diaginfo()
556                 #print self.data['message']
557                 if self.data['message']:
558                         message = PersistMessage(self.hostname, 
559                                                                  self.data['message'][0] % self.data['args'],
560                                                                  self.data['message'][1] % self.data['args'],
561                                                                  True, db='monitor_persistmessages',
562                                                                  ticket_id=ticket_id)
563                         if self.data['stage'] == "improvement":
564                                 message.reset()
565                         return message
566                 else:
567                         return None
568         
569         def getContacts(self):
570                 roles = self.data['email']
571
572                 if not config.mail and not config.debug and config.bcc:
573                         roles = ADMIN
574                 if config.mail and config.debug:
575                         roles = ADMIN
576
577                 # build targets
578                 contacts = []
579                 if ADMIN & roles:
580                         contacts += [config.email]
581                 if TECH & roles:
582                         #contacts += [TECHEMAIL % self.loginbase]
583                         contacts += plc.getTechEmails(self.loginbase)
584                 if PI & roles:
585                         #contacts += [PIEMAIL % self.loginbase]
586                         contacts += plc.getPIEmails(self.loginbase)
587                 if USER & roles:
588                         contacts += plc.getSliceUserEmails(self.loginbase)
589                         slices = plc.slices(self.loginbase)
590                         if len(slices) >= 1:
591                                 #for slice in slices:
592                                 #       contacts += [SLICEMAIL % slice]
593                                 print "SLIC: %20s : %d slices" % (self.loginbase, len(slices))
594                         else:
595                                 print "SLIC: %20s : 0 slices" % self.loginbase
596
597                 return contacts
598
599
600 class NodeRecord:
601         def __init__(self, hostname, target):
602                 self.hostname = hostname
603                 self.ticket = None
604                 self.target = target
605
606
607 class MonRecord(object):
608         def __init__(self, data):
609                 self.keys = data.keys()
610                 self.keys.sort()
611                 self.__dict__.update(data)
612                 return
613
614         def get(self):
615                 ret= {}
616                 for k in self.keys:
617                         ret[k] = self.__dict__[k]
618                 return ret
619
620         def __repr__(self):
621                 str = ""
622                 str += self.host + "\n"
623                 for k in self.keys:
624                         if "message" in k or "msg" in k:
625                                 continue
626                         if 'time' in k:
627                                 s_time=time.strftime("%Y/%m/%d %H:%M:%S", 
628                                                         time.gmtime(self.__dict__[k]))
629                                 str += "\t'%s' : %s\n" % (k, s_time)
630                         else:
631                                 str += "\t'%s' : %s\n" % (k, self.__dict__[k])
632                 str += "\t--"
633                 return str
634
635         def delField(self, field):
636                 if field in self.__dict__:
637                         del self.__dict__[field]
638                 
639                 if field in self.keys:
640                         for i in range(0,len(self.keys)):
641                                 if self.keys[i] == field:
642                                         del self.keys[i]
643                                         break
644
645 class Action(MonRecord):
646         def __init__(self, host, data):
647                 self.host = host
648                 MonRecord.__init__(self, data)
649                 return
650
651         def deltaDays(self, delta):
652                 t = datetime.fromtimestamp(self.__dict__['time'])
653                 d = t + timedelta(delta)
654                 self.__dict__['time'] = time.mktime(d.timetuple())
655                 
656 def node_end_record(node):
657         act_all = database.dbLoad("act_all")
658         if node not in act_all:
659                 del act_all
660                 return False
661
662         if len(act_all[node]) == 0:
663                 del act_all
664                 return False
665
666         pm = database.dbLoad("monitor_persistmessages")
667         if node not in pm:
668                 del pm
669                 return False
670         else:
671                 print "deleting node record"
672                 del pm[node]
673                 database.dbDump("monitor_persistmessages", pm)
674
675         a = Action(node, act_all[node][0])
676         a.delField('rt')
677         a.delField('found_rt_ticket')
678         a.delField('second-mail-at-oneweek')
679         a.delField('second-mail-at-twoweeks')
680         a.delField('first-found')
681         rec = a.get()
682         rec['action'] = ["close_rt"]
683         rec['category'] = "ALPHA"       # assume that it's up...
684         rec['stage'] = "monitor-end-record"
685         rec['ticket_id'] = None
686         rec['time'] = time.time() - 7*60*60*24
687         act_all[node].insert(0,rec)
688         database.dbDump("act_all", act_all)
689         del act_all
690         return True
691
692
693 class LogRoll:
694         def __init__(self, list=None):
695                 self.list = list
696                 if self.list == None:
697                         self.list = {}
698
699         def find(self, host, filter, timerange):
700                 if host not in self.list:
701                         return None
702
703                 host_log_list = self.list[host]
704                 for log in host_log_list:
705                         for key in filter.keys():
706                                 #print "searching key %s in log keys" % key
707                                 if key in log.keys:
708                                         #print "%s in log.keys" % key
709                                         cmp = re.compile(filter[key])
710                                         res = cmp.search(log.__getattribute__(key))
711                                         if res != None:
712                                                 #print "found match in log: %s  %s ~=~ %s" % (log, key, filter[key])
713                                                 if log.time > time.time() - timerange:
714                                                         print "returning log b/c it occured within time."
715                                                         return log
716                 return None
717                 
718
719         def get(self, host):
720                 if host in self.list:
721                         return self.list[host][0]
722                 else:
723                         return None
724
725         def add(self, log):
726                 if log.host not in self.list:
727                         self.list[log.host] = []
728
729                 self.list[log.host].insert(0,log)
730
731 class Log(MonRecord):
732         def __init__(self, host, data):
733                 self.host = host
734                 MonRecord.__init__(self, data)
735                 return
736
737         def __repr__(self):
738                 str = " "
739                 str += self.host + " : { "
740                 for k in self.keys:
741                         if "message" in k or "msg" in k:
742                                 continue
743                         if 'time' in k:
744                                 s_time=time.strftime("%Y/%m/%d %H:%M:%S", 
745                                                         time.gmtime(self.__dict__[k]))
746                                 #str += " '%s' : %s, " % (k, s_time)
747                         elif 'action' in k:
748                                 str += "'%s' : %s, " % (k, self.__dict__[k])
749                 str += "}"
750                 return str
751         
752
753 class Diagnose(MonRecord):
754         def __init__(self, host):
755                 self.host = host
756                 MonRecord.__init__(self, data)
757                 return
758
759
760
761 if __name__ == "__main__":
762         #r = RT()
763         #r.email("test", "body of test message", ['database@cs.princeton.edu'])
764         #from emailTxt import mailtxt
765         print "loaded"
766         #database.dbDump("persistmessages", {});
767         #args = {'url_list': 'http://www.planet-lab.org/bootcds/planet1.usb\n','hostname': 'planet1','hostname_list': ' blahblah -  days down\n'}
768         #m = PersistMessage("blue", "test 1", mailtxt.newdown_one[1] % args, True)
769         #m.send(['soltesz@cs.utk.edu'])
770         #m = PersistMessage("blue", "test 1 - part 2", mailtxt.newalphacd_one[1] % args, True)
771         # TRICK timer to thinking some time has passed.
772         #m.actiontracker.time = time.time() - 6*60*60*24
773         #m.send(['soltesz@cs.utk.edu'])