unified_model.py

   1 #!/usr/bin/python
   2
   3 from monitor import database
   4
   5 import plc
   6 import mailer
   7 import time
   8
   9 from model import *
  10 from const import *
  11 import util.file
  12 import config
  13
  14 def gethostlist(hostlist_file):
  15         return util.file.getListFromFile(hostlist_file)
  16
  17 def array_to_priority_map(array):
  18         """ Create a mapping where each entry of array is given a priority equal
  19         to its position in the array.  This is useful for subsequent use in the
  20         cmpMap() function."""
  21         map = {}
  22         count = 0
  23         for i in array:
  24                 map[i] = count
  25                 count += 1
  26         return map
  27
  28 def cmpValMap(v1, v2, map):
  29         if v1 in map and v2 in map and map[v1] < map[v2]:
  30                 return 1
  31         elif v1 in map and v2 in map and map[v1] > map[v2]:
  32                 return -1
  33         elif v1 in map and v2 in map:
  34                 return 0
  35         else:
  36                 raise Exception("No index %s or %s in map" % (v1, v2))
  37
  38 def cmpCategoryVal(v1, v2):
  39         # Terrible hack to manage migration to no more 'ALPHA' states.
  40         if v1 == 'ALPHA': v1 = "PROD"
  41         if v2 == 'ALPHA': v2 = "PROD"
  42         #map = array_to_priority_map([ None, 'PROD', 'ALPHA', 'OLDBOOTCD', 'UNKNOWN', 'FORCED', 'ERROR', ])
  43         map = array_to_priority_map([ None, 'ALPHA', 'PROD', 'OLDPROD', 'OLDBOOTCD', 'UNKNOWN', 'FORCED', 'ERROR', ])
  44         return cmpValMap(v1,v2,map)
  45
  46
  47 class PCU:
  48         def __init__(self, hostname):
  49                 self.hostname = hostname
  50
  51         def reboot(self):
  52                 return True
  53         def available(self):
  54                 return True
  55         def previous_attempt(self):
  56                 return True
  57         def setValidMapping(self):
  58                 pass
  59
  60 class Penalty:
  61         def __init__(self, key, valuepattern, action):
  62                 pass
  63
  64 class PenaltyMap:
  65         def __init__(self):
  66                 pass
  67
  68         # connect one penalty to another, in a FSM diagram.  After one
  69         #       condition/penalty is applied, move to the next phase.
  70
  71
  72 #fb = database.dbLoad("findbad")
  73
  74 class RT(object):
  75         def __init__(self, ticket_id = None):
  76                 self.ticket_id = ticket_id
  77                 if self.ticket_id:
  78                         print "getting ticket status",
  79                         self.status = mailer.getTicketStatus(self.ticket_id)
  80                         print self.status
  81
  82         def setTicketStatus(self, status):
  83                 mailer.setTicketStatus(self.ticket_id, status)
  84                 self.status = mailer.getTicketStatus(self.ticket_id)
  85                 return True
  86
  87         def getTicketStatus(self):
  88                 if not self.status:
  89                         self.status = mailer.getTicketStatus(self.ticket_id)
  90                 return self.status
  91
  92         def closeTicket(self):
  93                 mailer.closeTicketViaRT(self.ticket_id, "Ticket CLOSED automatically by SiteAssist.")
  94
  95         def email(self, subject, body, to):
  96                 self.ticket_id = mailer.emailViaRT(subject, body, to, self.ticket_id)
  97                 return self.ticket_id
  98
  99 class Message(object):
 100         def __init__(self, subject, message, via_rt=True, ticket_id=None, **kwargs):
 101                 self.via_rt = via_rt
 102                 self.subject = subject
 103                 self.message = message
 104                 self.rt = RT(ticket_id)
 105
 106         def send(self, to):
 107                 if self.via_rt:
 108                         return self.rt.email(self.subject, self.message, to)
 109                 else:
 110                         return mailer.email(self.subject, self.message, to)
 111
 112 class Recent(object):
 113         def __init__(self, withintime):
 114                 self.withintime = withintime
 115
 116                 try:
 117                         self.time = self.__getattribute__('time')
 118                 except:
 119                         self.time = time.time()- 7*24*60*60
 120
 121                 #self.time = time.time()
 122                 #self.action_taken = False
 123
 124         def isRecent(self):
 125                 if self.time + self.withintime < time.time():
 126                         self.action_taken = False
 127
 128                 if self.time + self.withintime > time.time() and self.action_taken:
 129                         return True
 130                 else:
 131                         return False
 132
 133         def unsetRecent(self):
 134                 self.action_taken = False
 135                 self.time = time.time()
 136                 return True
 137
 138         def setRecent(self):
 139                 self.action_taken = True
 140                 self.time = time.time()
 141                 return True
 142
 143 class PersistFlags(Recent):
 144         def __new__(typ, id, *args, **kwargs):
 145                 if 'db' in kwargs:
 146                         db = kwargs['db']
 147                         del kwargs['db']
 148                 else:
 149                         db = "persistflags"
 150
 151                 try:
 152                         pm = database.dbLoad(db)
 153                 except:
 154                         database.dbDump(db, {})
 155                         pm = database.dbLoad(db)
 156                 #print pm
 157                 if id in pm:
 158                         obj = pm[id]
 159                 else:
 160                         obj = super(PersistFlags, typ).__new__(typ, *args, **kwargs)
 161                         for key in kwargs.keys():
 162                                 obj.__setattr__(key, kwargs[key])
 163                         obj.time = time.time()
 164                         obj.action_taken = False
 165
 166                 obj.db = db
 167                 return obj
 168
 169         def __init__(self, id, withintime, **kwargs):
 170                 self.id = id
 171                 Recent.__init__(self, withintime)
 172
 173         def save(self):
 174                 pm = database.dbLoad(self.db)
 175                 pm[self.id] = self
 176                 database.dbDump(self.db, pm)
 177
 178         def resetFlag(self, name):
 179                 self.__setattr__(name, False)
 180
 181         def setFlag(self, name):
 182                 self.__setattr__(name, True)
 183
 184         def getFlag(self, name):
 185                 try:
 186                         return self.__getattribute__(name)
 187                 except:
 188                         self.__setattr__(name, False)
 189                         return False
 190
 191         def resetRecentFlag(self, name):
 192                 self.resetFlag(name)
 193                 self.unsetRecent()
 194
 195         def setRecentFlag(self, name):
 196                 self.setFlag(name)
 197                 self.setRecent()
 198
 199         def getRecentFlag(self, name):
 200                 # if recent and flag set -> true
 201                 # else false
 202                 try:
 203                         return self.isRecent() & self.__getattribute__(name)
 204                 except:
 205                         self.__setattr__(name, False)
 206                         return False
 207
 208         def checkattr(self, name):
 209                 try:
 210                         x = self.__getattribute__(name)
 211                         return True
 212                 except:
 213                         return False
 214
 215
 216 class PersistMessage(Message):
 217         def __new__(typ, id, subject, message, via_rt, **kwargs):
 218                 if 'db' in kwargs:
 219                         db = kwargs['db']
 220                 else:
 221                         db = "persistmessages"
 222
 223                 try:
 224                         pm = database.dbLoad(db)
 225                 except:
 226                         database.dbDump(db, {})
 227                         pm = database.dbLoad(db)
 228
 229                 #print pm
 230                 if id in pm:
 231                         #print "Using existing object"
 232                         obj = pm[id]
 233                 else:
 234                         #print "creating new object"
 235                         obj = super(PersistMessage, typ).__new__(typ, [id, subject, message, via_rt], **kwargs)
 236                         obj.id = id
 237                         obj.actiontracker = Recent(1*60*60*24)
 238                         obj.ticket_id = None
 239
 240                 if 'ticket_id' in kwargs and kwargs['ticket_id'] is not None:
 241                         obj.ticket_id = kwargs['ticket_id']
 242
 243                 obj.db = db
 244                 return obj
 245
 246         def __init__(self, id, subject, message, via_rt=True, **kwargs):
 247                 print "initializing object: %s" % self.ticket_id
 248                 self.id = id
 249                 Message.__init__(self, subject, message, via_rt, self.ticket_id)
 250
 251         def reset(self):
 252                 self.actiontracker.unsetRecent()
 253
 254         def save(self):
 255                 pm = database.dbLoad(self.db)
 256                 pm[self.id] = self
 257                 database.dbDump(self.db, pm)
 258
 259         def send(self, to):
 260                 if not self.actiontracker.isRecent():
 261                         self.ticket_id = Message.send(self, to)
 262                         self.actiontracker.setRecent()
 263                         self.save()
 264                 else:
 265                         # NOTE: only send a new message every week, regardless.
 266                         # NOTE: can cause thank-you messages to be lost, for instance when node comes back online within window.
 267                         print "Not sending to host b/c not within window of %s days" % (self.actiontracker.withintime // (60*60*24))
 268
 269 class MonitorMessage(object):
 270         def __new__(typ, id, *args, **kwargs):
 271                 if 'db' in kwargs:
 272                         db = kwargs['db']
 273                 else:
 274                         db = "monitormessages"
 275
 276                 try:
 277                         if 'reset' in kwargs and kwargs['reset'] == True:
 278                                 database.dbDump(db, {})
 279                         pm = database.dbLoad(db)
 280                 except:
 281                         database.dbDump(db, {})
 282                         pm = database.dbLoad(db)
 283
 284                 #print pm
 285                 if id in pm:
 286                         print "Using existing object"
 287                         obj = pm[id]
 288                 else:
 289                         print "creating new object"
 290                         obj = super(object, typ).__new__(typ, id, *args, **kwargs)
 291                         obj.id = id
 292                         obj.sp = PersistSitePenalty(id, 0)
 293
 294                 obj.db = db
 295                 return obj
 296
 297         def __init__(self, id, message):
 298                 pass
 299
 300
 301 class SitePenalty(object):
 302         penalty_map = []
 303         penalty_map.append( { 'name': 'noop',                   'enable'   : lambda host: None,
 304                                                                                                         'disable'  : lambda host: None } )
 305         penalty_map.append( { 'name': 'nocreate',               'enable'   : lambda host: plc.removeSliceCreation(host),
 306                                                                                                         'disable'  : lambda host: plc.enableSliceCreation(host) } )
 307         penalty_map.append( { 'name': 'suspendslices',  'enable'   : lambda host: plc.suspendSlices(host),
 308                                                                                                         'disable'  : lambda host: plc.enableSlices(host) } )
 309
 310         #def __init__(self, index=0, **kwargs):
 311         #       self.index = index
 312
 313         def get_penalties(self):
 314                 # TODO: get penalties actually applied to a node from PLC DB.
 315                 return [ n['name'] for n in SitePenalty.penalty_map ]
 316
 317         def increase(self):
 318                 self.index = self.index + 1
 319                 if self.index > len(SitePenalty.penalty_map)-1: self.index = len(SitePenalty.penalty_map)-1
 320                 return True
 321
 322         def decrease(self):
 323                 self.index = self.index - 1
 324                 if self.index < 0: self.index = 0
 325                 return True
 326
 327         def apply(self, host):
 328
 329                 for i in range(len(SitePenalty.penalty_map)-1,self.index,-1):
 330                         print "\tdisabling %s on %s" % (SitePenalty.penalty_map[i]['name'], host)
 331                         SitePenalty.penalty_map[i]['disable'](host)
 332
 333                 for i in range(0,self.index+1):
 334                         print "\tapplying %s on %s" % (SitePenalty.penalty_map[i]['name'], host)
 335                         SitePenalty.penalty_map[i]['enable'](host)
 336
 337                 return
 338
 339
 340
 341 class PersistSitePenalty(SitePenalty):
 342         def __new__(typ, id, index, **kwargs):
 343                 if 'db' in kwargs:
 344                         db = kwargs['db']
 345                 else:
 346                         db = "persistpenalties"
 347
 348                 try:
 349                         if 'reset' in kwargs and kwargs['reset'] == True:
 350                                 database.dbDump(db, {})
 351                         pm = database.dbLoad(db)
 352                 except:
 353                         database.dbDump(db, {})
 354                         pm = database.dbLoad(db)
 355
 356                 #print pm
 357                 if id in pm:
 358                         print "PersistSitePenalty Using existing object"
 359                         obj = pm[id]
 360                 else:
 361                         print "creating new object"
 362                         obj = super(PersistSitePenalty, typ).__new__(typ, [index], **kwargs)
 363                         obj.id = id
 364                         obj.index = index
 365
 366                 obj.db = db
 367                 return obj
 368
 369         def __init__(self, id, index, **kwargs):
 370                 self.id = id
 371
 372         def save(self):
 373                 pm = database.dbLoad(self.db)
 374                 pm[self.id] = self
 375                 database.dbDump(self.db, pm)
 376
 377
 378 class Target:
 379         """
 380                 Each host has a target set of attributes.  Some may be set manually,
 381                 or others are set globally for the preferred target.
 382
 383                 For instance:
 384                         All nodes in the Alpha or Beta group would have constraints like:
 385                                 [ { 'state' : 'BOOT', 'kernel' : '2.6.22' } ]
 386         """
 387         def __init__(self, constraints):
 388                 self.constraints = constraints
 389
 390         def verify(self, data):
 391                 """
 392                         self.constraints is a list of key, value pairs.
 393                         # [ {... : ...}==AND , ... , ... , ] == OR
 394                 """
 395                 con_or_true = False
 396                 for con in self.constraints:
 397                         #print "con: %s" % con
 398                         con_and_true = True
 399                         for key in con.keys():
 400                                 #print "looking at key: %s" % key
 401                                 if key in data:
 402                                         #print "%s %s" % (con[key], data[key])
 403                                         con_and_true = con_and_true & (con[key] in data[key])
 404                                 elif key not in data:
 405                                         print "missing key %s" % key
 406                                         con_and_true = False
 407
 408                         con_or_true = con_or_true | con_and_true
 409
 410                 return con_or_true
 411
 412 class Record(object):
 413
 414         def __init__(self, hostname, data):
 415                 self.hostname = hostname
 416                 self.data = data
 417                 self.plcdb_hn2lb = database.dbLoad("plcdb_hn2lb")
 418                 self.loginbase = self.plcdb_hn2lb[self.hostname]
 419                 return
 420
 421
 422         def stageIswaitforever(self):
 423                 if 'waitforever' in self.data['stage']:
 424                         return True
 425                 else:
 426                         return False
 427
 428         def severity(self):
 429                 category = self.data['category']
 430                 prev_category = self.data['prev_category']
 431                 print "SEVERITY: ", category, prev_category
 432                 try:
 433                         print "SEVERITY state: ", self.data['state'], self.data['prev_state']
 434                 except:
 435                         print "SEVERITY state: unknown unknown"
 436                 val = cmpCategoryVal(category, prev_category)
 437                 return val
 438
 439         def improved(self):
 440                 return self.severity() > 0
 441
 442         def end_record(self):
 443                 return node_end_record(self.hostname)
 444
 445         def reset_stage(self):
 446                 self.data['stage'] = 'findbad'
 447                 return True
 448
 449         def getCategory(self):
 450                 return self.data['category'].lower()
 451
 452         def getState(self):
 453                 return self.data['state'].lower()
 454
 455         def getDaysDown(cls, diag_record):
 456                 daysdown = -1
 457                 if diag_record['comonstats']['uptime'] != "null" and diag_record['comonstats']['uptime'] != "-1":
 458                         daysdown = - int(float(diag_record['comonstats']['uptime'])) // (60*60*24)
 459                 #elif diag_record['comonstats']['sshstatus'] != "null":
 460                 #       daysdown = int(diag_record['comonstats']['sshstatus']) // (60*60*24)
 461                 #elif diag_record['comonstats']['lastcotop'] != "null":
 462                 #       daysdown = int(diag_record['comonstats']['lastcotop']) // (60*60*24)
 463                 else:
 464                         now = time.time()
 465                         last_contact = diag_record['plcnode']['last_contact']
 466                         if last_contact == None:
 467                                 # the node has never been up, so give it a break
 468                                 daysdown = -1
 469                         else:
 470                                 diff = now - last_contact
 471                                 daysdown = diff // (60*60*24)
 472                 return daysdown
 473         getDaysDown = classmethod(getDaysDown)
 474
 475         def getStrDaysDown(cls, diag_record):
 476                 daysdown = "unknown"
 477                 last_contact = diag_record['plcnode']['last_contact']
 478                 date_created = diag_record['plcnode']['date_created']
 479
 480                 if      diag_record['comonstats']['uptime'] != "null" and \
 481                         diag_record['comonstats']['uptime'] != "-1":
 482                         daysdown = int(float(diag_record['comonstats']['uptime'])) // (60*60*24)
 483                         daysdown = "%d days up" % daysdown
 484
 485                 elif last_contact is None:
 486                         if date_created is not None:
 487                                 now = time.time()
 488                                 diff = now - date_created
 489                                 daysdown = diff // (60*60*24)
 490                                 daysdown = "Never contacted PLC, created %s days ago" % daysdown
 491                         else:
 492                                 daysdown = "Never contacted PLC"
 493                 else:
 494                         now = time.time()
 495                         diff = now - last_contact
 496                         daysdown = diff // (60*60*24)
 497                         daysdown = "%s days down" % daysdown
 498                 return daysdown
 499         getStrDaysDown = classmethod(getStrDaysDown)
 500
 501         #def getStrDaysDown(cls, diag_record):
 502         #       daysdown = cls.getDaysDown(diag_record)
 503         #       if daysdown > 0:
 504         #               return "%d days down"%daysdown
 505         #       elif daysdown == -1:
 506         #               return "Never online"
 507         #       else:
 508         #               return "%d days up"% -daysdown
 509         #getStrDaysDown = classmethod(getStrDaysDown)
 510
 511         def takeAction(self, index=0):
 512                 pp = PersistSitePenalty(self.hostname, 0, db='persistpenalty_hostnames')
 513                 if 'improvement' in self.data['stage'] or self.improved() or \
 514                         'monitor-end-record' in self.data['stage']:
 515                         print "takeAction: decreasing penalty for %s"%self.hostname
 516                         pp.decrease()
 517                         pp.decrease()
 518                 else:
 519                         print "takeAction: increasing penalty for %s"%self.hostname
 520                         pp.increase()
 521                 print "takeAction: applying penalty to %s as index %s"% (self.hostname, index)
 522                 pp.index = index
 523                 pp.apply(self.hostname)
 524                 pp.save()
 525
 526         def _format_diaginfo(self):
 527                 info = self.data['info']
 528                 print "FORMAT : STAGE: ", self.data['stage']
 529                 if self.data['stage'] == 'monitor-end-record':
 530                         if info[2] == "ALPHA": info = (info[0], info[1], "PROD")
 531                         hlist = "    %s went from '%s' to '%s'\n" % (info[0], info[1], info[2])
 532                 else:
 533                         hlist = "    %s %s - %s\n" % (info[0], info[2], info[1]) #(node,ver,daysdn)
 534                 return hlist
 535         def saveAction(self):
 536                 if 'save-act-all' in self.data and self.data['save-act-all'] == True:
 537                         return True
 538                 else:
 539                         return False
 540
 541         def getMessage(self, ticket_id=None):
 542                 self.data['args']['hostname'] = self.hostname
 543                 self.data['args']['loginbase'] = self.loginbase
 544                 self.data['args']['hostname_list'] = self._format_diaginfo()
 545                 #print self.data['message']
 546                 if self.data['message']:
 547                         message = PersistMessage(self.hostname,
 548                                                                  self.data['message'][0] % self.data['args'],
 549                                                                  self.data['message'][1] % self.data['args'],
 550                                                                  True, db='monitor_persistmessages',
 551                                                                  ticket_id=ticket_id)
 552                         if self.data['stage'] == "improvement":
 553                                 message.reset()
 554                         return message
 555                 else:
 556                         return None
 557
 558         def getContacts(self):
 559                 roles = self.data['email']
 560
 561                 if not config.mail and not config.debug and config.bcc:
 562                         roles = ADMIN
 563                 if config.mail and config.debug:
 564                         roles = ADMIN
 565
 566                 # build targets
 567                 contacts = []
 568                 if ADMIN & roles:
 569                         contacts += [config.email]
 570                 if TECH & roles:
 571                         #contacts += [TECHEMAIL % self.loginbase]
 572                         contacts += plc.getTechEmails(self.loginbase)
 573                 if PI & roles:
 574                         #contacts += [PIEMAIL % self.loginbase]
 575                         contacts += plc.getPIEmails(self.loginbase)
 576                 if USER & roles:
 577                         contacts += plc.getSliceUserEmails(self.loginbase)
 578                         slices = plc.slices(self.loginbase)
 579                         if len(slices) >= 1:
 580                                 #for slice in slices:
 581                                 #       contacts += [SLICEMAIL % slice]
 582                                 print "SLIC: %20s : %d slices" % (self.loginbase, len(slices))
 583                         else:
 584                                 print "SLIC: %20s : 0 slices" % self.loginbase
 585
 586                 return contacts
 587
 588
 589 class NodeRecord:
 590         def __init__(self, hostname, target):
 591                 self.hostname = hostname
 592                 self.ticket = None
 593                 self.target = target
 594                 #if hostname in fb['nodes']:
 595                 #       self.data = fb['nodes'][hostname]['values']
 596                 #else:
 597                 #       raise Exception("Hostname not in scan database")
 598
 599         def stageIswaitforever(self):
 600                 if 'waitforever' in self.data['stage']:
 601                         return True
 602                 else:
 603                         return False
 604
 605         def severity(self):
 606                 category = self.data['category']
 607                 prev_category = self.data['prev_category']
 608                 print "IMPROVED: ", category, prev_category
 609                 val = cmpCategoryVal(category, prev_category)
 610                 return val
 611
 612         def improved(self):
 613                 return self.severity() > 0
 614
 615         def end_record(self):
 616                 return node_end_record(self.hostname)
 617
 618         def reset_stage(self):
 619                 self.data['stage'] = 'findbad'
 620                 return True
 621
 622         def open_tickets(self):
 623                 if self.ticket and self.ticket.status['status'] == 'open':
 624                         return 1
 625                 return 0
 626         def setIntrospect(self):
 627                 pass
 628
 629         def email_notice(self):
 630                 message = self._get_message_for_condition()
 631                 message.send(self._get_contacts_for_condition())
 632                 return True
 633         def close_ticket(self):
 634                 if self.ticket:
 635                         self.ticket.closeTicket()
 636
 637         def exempt_from_penalties(self):
 638                 bl = database.dbLoad("l_blacklist")
 639                 return self.hostname in bl
 640
 641         def penalties(self):
 642                 return []
 643         def escellate_penalty(self):
 644                 return True
 645         def reduce_penalty(self):
 646                 return True
 647
 648
 649         def atTarget(self):
 650                 return self.target.verify(self.data)
 651
 652         def _get_condition(self):
 653                 return self.data['category'].lower()
 654
 655         def _get_stage(self):
 656                 "improvement"
 657                 "firstnotice_noop"
 658                 "secondnotice_noslicecreation"
 659                 "thirdnotice_disableslices"
 660
 661                 delta = current_time - self.data['time']
 662
 663         def _get_message_for_condition(self):
 664                 pass
 665         def _get_contacts_for_condition(self):
 666                 pass
 667
 668 class Action(MonRecord):
 669         def __init__(self, host, data):
 670                 self.host = host
 671                 MonRecord.__init__(self, data)
 672                 return
 673
 674         def deltaDays(self, delta):
 675                 t = datetime.fromtimestamp(self.__dict__['time'])
 676                 d = t + timedelta(delta)
 677                 self.__dict__['time'] = time.mktime(d.timetuple())
 678
 679 def node_end_record(node):
 680         act_all = database.dbLoad("act_all")
 681         if node not in act_all:
 682                 del act_all
 683                 return False
 684
 685         if len(act_all[node]) == 0:
 686                 del act_all
 687                 return False
 688
 689         pm = database.dbLoad("monitor_persistmessages")
 690         if node not in pm:
 691                 del pm
 692                 return False
 693         else:
 694                 print "deleting node record"
 695                 del pm[node]
 696                 database.dbDump("monitor_persistmessages", pm)
 697
 698         a = Action(node, act_all[node][0])
 699         a.delField('rt')
 700         a.delField('found_rt_ticket')
 701         a.delField('second-mail-at-oneweek')
 702         a.delField('second-mail-at-twoweeks')
 703         a.delField('first-found')
 704         rec = a.get()
 705         rec['action'] = ["close_rt"]
 706         rec['category'] = "ALPHA"       # assume that it's up...
 707         rec['stage'] = "monitor-end-record"
 708         rec['ticket_id'] = None
 709         rec['time'] = time.time() - 7*60*60*24
 710         act_all[node].insert(0,rec)
 711         database.dbDump("act_all", act_all)
 712         del act_all
 713         return True
 714
 715 if __name__ == "__main__":
 716         #r = RT()
 717         #r.email("test", "body of test message", ['database@cs.princeton.edu'])
 718         #from emailTxt import mailtxt
 719         print "loaded"
 720         #database.dbDump("persistmessages", {});
 721         #args = {'url_list': 'http://www.planet-lab.org/bootcds/planet1.usb\n','hostname': 'planet1','hostname_list': ' blahblah -  days down\n'}
 722         #m = PersistMessage("blue", "test 1", mailtxt.newdown_one[1] % args, True)
 723         #m.send(['soltesz@cs.utk.edu'])
 724         #m = PersistMessage("blue", "test 1 - part 2", mailtxt.newalphacd_one[1] % args, True)
 725         # TRICK timer to thinking some time has passed.
 726         #m.actiontracker.time = time.time() - 6*60*60*24
 727         #m.send(['soltesz@cs.utk.edu'])