web/MonitorWeb/monitorweb/controllers.py

   1 import turbogears as tg
   2 from turbogears import controllers, expose, flash, exception_handler, redirect
   3 from turbogears import widgets
   4 from cherrypy import request, response
   5 import cherrypy
   6 # from monitorweb import model
   7 # import logging
   8 # log = logging.getLogger("monitorweb.controllers")
   9 import re
  10 import os
  11 from monitor.database.info.model import *
  12 #from monitor.database.zabbixapi.model import *
  13 from monitor_xmlrpc import MonitorXmlrpcServer
  14
  15 from monitor import util
  16 from monitor import reboot
  17 from monitor import bootman
  18 from monitor import scanapi
  19 from monitor import config
  20 import time
  21
  22 from monitor.wrapper.plccache import plcdb_hn2lb as site_hn2lb
  23
  24 from monitorweb.templates.links import *
  25
  26 class ObjectQueryFields(widgets.WidgetsList):
  27         """The WidgetsList defines the fields of the form."""
  28         pass
  29
  30
  31
  32 class NodeQueryFields(widgets.WidgetsList):
  33         """The WidgetsList defines the fields of the form."""
  34
  35         object = widgets.RadioButtonList(label="Query Type", options=[('nodes', 'All Nodes'),
  36                                                                                                                           ('nodehistory', 'Single Node History'),
  37                                                                                                                           #('sites', 'All Sites'),
  38                                                                                                                           #('sitehistory', 'Single Site History'),
  39                                                                                                                           ], default="nodes")
  40         nodehistory_hostname = widgets.TextField(label="Hostname Node History", attrs={'size':30})
  41
  42         hostname = widgets.CheckBox(label="Hostname")
  43         firewall = widgets.CheckBox(label="Firewall?")
  44         fs_status = widgets.CheckBox(label="Filesystem Status")
  45         ssh_status = widgets.CheckBox(label="SSH Status")
  46         ssh_error = widgets.CheckBox(label="SSH Errors")
  47         dns_status = widgets.CheckBox(label="DNS Status")
  48         iptables_status = widgets.CheckBox(label="IP Tables Status")
  49         nm_status = widgets.CheckBox(label="NM Status")
  50         princeton_comon_dir = widgets.CheckBox(label="CoMon Dir")
  51         princeton_comon_running = widgets.CheckBox(label="CoMon Running")
  52         princeton_comon_procs = widgets.CheckBox(label="CoMon Processes")
  53         external_dns_status = widgets.CheckBox(label="Hostname Resolves?")
  54         kernel_version = widgets.CheckBox(label="Kernel")
  55         bootcd_version = widgets.CheckBox(label="BootCD")
  56         boot_server = widgets.CheckBox(label="Boot Server")
  57         observed_status = widgets.CheckBox(label="Observed Status")
  58         uptime = widgets.CheckBox(label="Uptime")
  59         traceroute = widgets.CheckBox(label="Traceroute")
  60         port_status = widgets.CheckBox(label="Port Status")
  61         rpms = widgets.CheckBox(label="RPM")
  62         rpmvalue = widgets.TextField(label="RPM Pattern")
  63
  64 class QueryForm(widgets.TableForm):
  65     template = """
  66     <form xmlns:py="http://purl.org/kid/ns#"
  67         id="queryform"
  68         name="${name}"
  69         action="${action}"
  70         method="${method}"
  71         class="tableform"
  72         py:attrs="form_attrs"
  73     >
  74         <div py:for="field in hidden_fields"
  75             py:replace="field.display(value_for(field), **params_for(field))"
  76         />
  77         <table border="0" cellspacing="0" cellpadding="2" py:attrs="table_attrs">
  78             <tr py:for="i, field in enumerate(fields)"
  79                 class="${i%2 and 'odd' or 'even'}"
  80             >
  81                 <th>
  82                     <label class="fieldlabel" for="${field.field_id}" py:content="field.label" />
  83                 </th>
  84                 <td>
  85                     <span py:replace="field.display(value_for(field), **params_for(field))" />
  86                     <span py:if="error_for(field)" class="fielderror" py:content="error_for(field)" />
  87                     <span py:if="field.help_text" class="fieldhelp" py:content="field.help_text" />
  88                 </td>
  89             </tr>
  90             <tr>
  91                 <td>&#160;</td>
  92                 <td py:content="submit.display(submit_text)" />
  93             </tr>
  94         </table>
  95     </form>
  96         """
  97
  98 def getNodeQueryForm():
  99         return QueryForm(fields=NodeQueryFields(), action="query")
 100
 101 # make it easier group objects without invoking the elixir auto-write feature.
 102 class aggregate: pass
 103
 104
 105 def query_to_dict(query):
 106         """ take a url query string and chop it up """
 107         val = {}
 108         query_fields = query.split('&')
 109         for f in query_fields:
 110                 (k,v) = urllib.splitvalue(f)
 111                 val[k] = v
 112
 113         return val
 114
 115 def format_ports(data, pcumodel=None):
 116         retval = []
 117         filtered_length=0
 118
 119         if pcumodel:
 120                 supported_ports=reboot.model_to_object(pcumodel).supported_ports
 121         else:
 122                 # ports of a production node
 123                 supported_ports=[22,80,806]
 124
 125         if data and len(data.keys()) > 0 :
 126                 for port in supported_ports:
 127                         try:
 128                                 state = data[str(port)]
 129                         except:
 130                                 state = "unknown"
 131
 132                         if state == "filtered":
 133                                 filtered_length += 1
 134
 135                         retval.append( (port, state) )
 136
 137         if retval == []:
 138                 retval = [( "Closed/Filtered", "state" )]
 139
 140         if filtered_length == len(supported_ports):
 141                 retval = [( "All Filtered", "state" )]
 142
 143         return retval
 144
 145 def format_pcu_shortstatus(pcu):
 146         status = "error"
 147         if pcu:
 148                 if pcu.reboot_trial_status == str(0):
 149                         status = "Ok"
 150                 elif pcu.reboot_trial_status == "NetDown" or pcu.reboot_trial_status == "Not_Run":
 151                         status = pcu.reboot_trial_status
 152                 else:
 153                         status = "error"
 154
 155         return status
 156
 157 def prep_pcu_for_display(pcu):
 158         agg = aggregate()
 159         agg.pcu = pcu
 160
 161         try:
 162                 agg.loginbase = PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base']
 163         except:
 164                 agg.loginbase = "unknown"
 165
 166         agg.pcuhist = HistoryPCURecord.query.get(pcu.plc_pcuid)
 167
 168         agg.ports = format_ports(pcu.port_status, pcu.plc_pcu_stats['model'])
 169         agg.status = format_pcu_shortstatus(pcu)
 170
 171         #print pcu.entry_complete
 172         agg.entry_complete_str = pcu.entry_complete
 173         #pcu.entry_complete_str += "".join([ f[0] for f in pcu.entry_complete.split() ])
 174         if pcu.dns_status == "NOHOSTNAME":
 175                 agg.dns_short_status = 'NoHost'
 176         elif pcu.dns_status == "DNS-OK":
 177                 agg.dns_short_status = 'Ok'
 178         elif pcu.dns_status == "DNS-NOENTRY":
 179                 agg.dns_short_status = 'NoEntry'
 180         elif pcu.dns_status == "NO-DNS-OR-IP":
 181                 agg.dns_short_status = 'NoHostOrIP'
 182         elif pcu.dns_status == "DNS-MISMATCH":
 183                 agg.dns_short_status = 'Mismatch'
 184         return agg
 185
 186 class ActionListWidget(widgets.Widget):
 187         pass
 188
 189 class NodeWidget(widgets.Widget):
 190         pass
 191
 192 def prep_nodehist(node):
 193         agg = aggregate()
 194         agg.node = node
 195         agg.loginbase = "unknown"
 196         try:
 197                 agg.loginbase = PlcSite.query.get(node.plc_siteid).plc_site_stats['login_base']
 198         except:
 199                 agg.loginbase = "exception"
 200
 201
 202         return agg
 203
 204 def prep_node_for_display(node, pcuhash=None, preppcu=True, asofdate=None):
 205         agg = aggregate()
 206         agg.node = node
 207
 208         if node.plc_pcuid and preppcu:
 209                 if pcuhash:
 210                         pcu = pcuhash[node.plc_pcuid]
 211                 else:
 212                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=node.plc_pcuid)
 213
 214                 if pcu:
 215                         agg.pcu_status = pcu.reboot_trial_status
 216                         agg.pcu_short_status = format_pcu_shortstatus(pcu)
 217                         agg.pcu = prep_pcu_for_display(pcu)
 218                 else:
 219                         agg.pcu_short_status = "none"
 220                         agg.pcu_status = "nodata"
 221                         agg.pcu = None
 222
 223         else:
 224                 agg.pcu_status = "nopcu"
 225                 agg.pcu_short_status = "none"
 226                 agg.pcu = None
 227
 228
 229         if node.kernel_version:
 230                 agg.kernel = node.kernel_version.split()[2]
 231         else:
 232                 agg.kernel = ""
 233
 234         try:
 235                 agg.loginbase = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 236         except:
 237                 agg.loginbase = "unknown"
 238
 239         if agg.loginbase:
 240                 agg.site = HistorySiteRecord.by_loginbase(agg.loginbase)
 241
 242                 if asofdate:
 243                         agg.site = agg.site.get_as_of(asofdate)
 244
 245                 if agg.site is None:
 246                         # TODO: need a cleaner fix for this...
 247                         agg.site = HistorySiteRecord.by_loginbase("pl")
 248                         if not agg.site:
 249                                 agg.site = HistorySiteRecord.by_loginbase("ple")
 250
 251         agg.history = HistoryNodeRecord.by_hostname(node.hostname)
 252         if asofdate:
 253                 agg.history = agg.history.get_as_of(asofdate)
 254
 255         agg.ports = format_ports(node.port_status)
 256
 257         try:
 258                 exists = node.plc_node_stats['last_contact']
 259         except:
 260                 # TODO: this should not assign to the fb object!
 261                 node.plc_node_stats = {'last_contact' : None}
 262
 263         return agg
 264
 265
 266 class Root(controllers.RootController, MonitorXmlrpcServer):
 267         @expose(template="monitorweb.templates.welcome")
 268         def index(self):
 269                 # log.debug("Happy TurboGears Controller Responding For Duty")
 270                 flash("Welcome To MyOps!")
 271                 return dict(now=time.ctime())
 272
 273         @expose(template="monitorweb.templates.nodelist", allow_json=True)
 274         def node3(self, filter=None):
 275                 nhquery = HistoryNodeRecord.query.all()
 276                 query = []
 277                 for nh in nhquery:
 278                         if filter:
 279                                 if nh.status == filter:
 280                                         query.append(nh)
 281                         else:
 282                                 query.append(nh)
 283
 284                 rquery=[]
 285                 for q in query:
 286                         fb = FindbadNodeRecord.get_latest_by(hostname=q.hostname)
 287                         rquery.append(fb)
 288
 289                 return dict(now=time.ctime(), query=rquery)
 290
 291         def node_query(self, filter):
 292                 nhquery = HistoryNodeRecord.query.all()
 293                 query = []
 294                 for nh in nhquery:
 295                         if filter:
 296                                 if nh.status == filter:
 297                                         query.append(nh)
 298                         else:
 299                                 query.append(nh)
 300
 301                 rquery=[]
 302                 for q in query:
 303                         fb = FindbadNodeRecord.get_latest_by(hostname=q.hostname)
 304                         agg = prep_node_for_display(fb)
 305                         rquery.append(agg)
 306                 return rquery
 307
 308         @expose("cheetah:monitorweb.templates.nodelist_plain", as_format="plain",
 309                 accept_format="text/plain", content_type="text/plain")
 310         @expose(template="monitorweb.templates.nodelist", allow_json=True)
 311         def node2(self, filter=None):
 312                 rquery=self.node_query(filter)
 313                 widget = NodeWidget(template='monitorweb.templates.node_template')
 314                 return dict(now=time.ctime(), query=rquery, nodewidget=widget)
 315
 316         @expose("cheetah:monitorweb.templates.query_plain", as_format="plain",
 317                 accept_format="text/plain", content_type="text/plain")
 318         @expose(template="monitorweb.templates.query", allow_json=True)
 319         def query(self, **data):
 320                 query = []
 321
 322                 for k in data:
 323                         print k, data[k]
 324
 325                 fbquery = None
 326
 327                 if 'object' in data and data['object'] == "nodes":
 328                         fbquery = FindbadNodeRecord.get_all_latest()
 329                 elif 'object' in data and data['object'] == "nodehistory":
 330                         hostname = data['nodehistory_hostname']
 331                         data['date_checked'] = 'date_checked'
 332                         fbrecord = FindbadNodeRecord.get_by(hostname=hostname)
 333                         fbquery = fbrecord.versions[-500:]
 334
 335                 if fbquery:
 336                         for node in fbquery:
 337                                 # NOTE: reformat some fields.
 338                                 if type(node) is not type(FindbadNodeRecord):
 339                                         agg = node.__dict__.copy()
 340                                 else:
 341                                         agg = node.to_dict()
 342                                 agg.update(agg['plc_node_stats'])
 343                                 if agg['kernel_version']:
 344                                         agg['kernel_version'] = agg['kernel_version'].split()[2]
 345                                 if 'traceroute' in data and agg['traceroute']:
 346                                         agg['traceroute'] = "<pre>" + agg['traceroute'] + "</pre>"
 347                                 if 'rpmvalue' in data and 'rpms' in data:
 348                                         if agg['rpms']:
 349                                                 rpm_list = agg['rpms'].split()
 350                                                 rpm_list = filter(lambda x: data['rpmvalue'] in x, rpm_list)
 351                                                 agg['rpms'] = " ".join(rpm_list)
 352
 353                                 query.append(agg)
 354
 355                 fields=data.copy()
 356
 357                 try:
 358                         del fields['object']
 359                         del fields['rpmvalue']
 360                         del fields['nodehistory_hostname']
 361                 except: pass
 362                 return dict(now=time.ctime(), query=query, fields=fields, data=data, queryform=getNodeQueryForm())
 363
 364         @expose(template="monitorweb.templates.nodefast", allow_json=True)
 365         def node(self, filter=None):
 366                 nhquery = HistoryNodeRecord.query.all()
 367                 query = []
 368                 for nh in nhquery:
 369                         if filter:
 370                                 if nh.status == filter:
 371                                         agg = prep_nodehist(nh)
 372                                         query.append(agg)
 373                         else:
 374                                 agg = prep_nodehist(nh)
 375                                 query.append(agg)
 376
 377                 return dict(now=time.ctime(), query=query)
 378
 379         @expose(template="monitorweb.templates.nodelist")
 380         def nodeslow(self, filter='boot'):
 381                 print "NODE------------------"
 382                 print "befor-len: ", len( [ i for i in session] )
 383                 session.flush(); session.clear()
 384                 print "after-len: ", len( [ i for i in session] )
 385                 fbquery = FindbadNodeRecord.get_all_latest()
 386                 query = []
 387                 filtercount = {'down' : 0, 'boot': 0, 'debug' : 0, 'diagnose' : 0, 'disabled': 0,
 388                                                 'neverboot' : 0, 'pending' : 0, 'all' : 0, None : 0}
 389                 for node in fbquery:
 390                         # NOTE: reformat some fields.
 391                         agg = prep_node_for_display(node)
 392
 393                         if not agg.history:
 394                                 continue
 395
 396                         if agg.history.status in ['down', 'offline']:
 397                                 if node.plc_node_stats and node.plc_node_stats['last_contact'] != None:
 398                                         filtercount['down'] += 1
 399                                 else:
 400                                         filtercount['neverboot'] += 1
 401                         elif agg.history.status in ['good', 'online']:
 402                                 filtercount['boot'] += 1
 403                         elif agg.history.status in ['debug', 'monitordebug']:
 404                                 filtercount['debug'] += 1
 405                         else:
 406                                 if filtercount.has_key(agg.history.status):
 407                                         filtercount[agg.history.status] += 1
 408
 409
 410                         # NOTE: apply filter
 411                         if filter == "neverboot":
 412                                 if not node.plc_node_stats or node.plc_node_stats['last_contact'] == None:
 413                                         query.append(agg)
 414                         elif filter == "all":
 415                                 query.append(agg)
 416                         elif filter == agg.history.status:
 417                                 query.append(agg)
 418                         elif filter == 'boot':
 419                                 query.append(agg)
 420
 421
 422                 widget = NodeWidget(template='monitorweb.templates.node_template')
 423                 return dict(now=time.ctime(), query=query, fc=filtercount, nodewidget=widget)
 424
 425         def nodeaction_handler(self, tg_exceptions=None):
 426                 """Handle any kind of error."""
 427                 print "NODEACTION_HANDLER------------------"
 428
 429                 if 'pcuid' in request.params:
 430                         pcuid = request.params['pcuid']
 431                 else:
 432                         refurl = request.headers.get("Referer",link("pcu"))
 433                         print refurl
 434
 435                         # TODO: do this more intelligently...
 436                         uri_fields = urllib.splitquery(refurl)
 437                         if uri_fields[1] is not None:
 438                                 val = query_to_dict(uri_fields[1])
 439                                 if 'pcuid' in val:
 440                                         pcuid = val['pcuid']
 441                                 elif 'hostname' in val:
 442                                         pcuid = FindbadNodeRecord.get_latest_by(hostname=val['hostname']).plc_pcuid
 443                                 else:
 444                                         pcuid=None
 445                         else:
 446                                 pcuid=None
 447
 448                 cherry_trail = cherrypy._cputil.get_object_trail()
 449                 for i in cherry_trail:
 450                         print "trail: ", i
 451
 452                 print pcuid
 453                 return self.pcuview(None, pcuid, **dict(exceptions=tg_exceptions))
 454
 455         def nodeaction(self, **data):
 456                 print "NODEACTION------------------"
 457                 for item in data.keys():
 458                         print "%s %s" % ( item, data[item] )
 459
 460                 if 'hostname' in data:
 461                         hostname = data['hostname']
 462                 else:
 463                         flash("No hostname given in submitted data")
 464                         return
 465
 466                 if 'submit' in data or 'type' in data:
 467                         try:
 468                                 action = data['submit']
 469                         except:
 470                                 action = data['type']
 471                 else:
 472                         flash("No submit action given in submitted data")
 473                         return
 474
 475                 if action == "Reboot":
 476                         print "REBOOT: %s" % hostname
 477                         ret = reboot.reboot_str(str(hostname))
 478                         print ret
 479                         if ret: raise RuntimeError("Error using PCU: " + str(ret))
 480                         flash("Reboot appeared to work.  Allow at most 5 minutes.  Then run ExternalScan to check current status.")
 481
 482                 elif action == "ExternalScan":
 483                         scanapi.externalprobe(str(hostname))
 484                         flash("External Scan Successful!")
 485                 elif action == "InternalScan":
 486                         scanapi.internalprobe(str(hostname))
 487                         flash("Internal Scan Successful!")
 488                 else:
 489                         # unknown action
 490                         raise RuntimeError("Unknown action given")
 491                 return
 492
 493         @expose(template="monitorweb.templates.simpleview")
 494         def simpleview(self, **data):
 495                 return self.pre_view(**data)
 496
 497         @expose(template="monitorweb.templates.simpleview")
 498         def pcuview(self, **data):
 499                 return self.pre_view(**data)
 500
 501         @expose(template="monitorweb.templates.detailview")
 502         def detailview(self, **data):
 503                 return self.pre_view(**data)
 504
 505
 506         def pre_view(self, **data):
 507                 session.flush(); session.clear()
 508
 509                 loginbase=None
 510                 loginbase_list=[]
 511                 hostname=None
 512                 pcuid=None
 513                 since=20
 514                 # if objtype is not None, then treat 'hostname' or 'loginbase' as a search pattern
 515                 objtype=None
 516
 517                 exceptions = None
 518                 sitequery=[]
 519                 nodequery=[]
 520                 pcuquery=[]
 521                 actions=[]
 522                 actions_list=[]
 523
 524                 for key in data:
 525                         print key, data[key]
 526
 527                 if 'query' in data:
 528                         obj = data['query']
 529                         fields = obj.split(":")
 530                         if len(fields) > 1:
 531                                 objtype = fields[0]
 532                                 obj = fields[1].replace("*", "%")
 533                                 print "obj: %s"% obj
 534
 535                         if len(obj.split(".")) > 1 or objtype == "node":
 536                                 hostname = obj
 537                         else:
 538                                 loginbase = obj
 539
 540                 if 'loginbase' in data:
 541                         loginbase = data['loginbase']
 542
 543                 if 'hostname' in data:
 544                         hostname = data['hostname']
 545
 546                 if 'pcuid' in data:
 547                         try: pcuid = int(data['pcuid'])
 548                         except: pcuid = None
 549
 550                 if 'since' in data:
 551                         try: since = int(since)
 552                         except: since = 20
 553
 554                 if pcuid:
 555                         print "pcuid: %s" % pcuid
 556                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=pcuid)
 557                         loginbase_list += [ PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base'] ]
 558
 559                 if hostname:
 560                         if not objtype:
 561                                 nodes = [ FindbadNodeRecord.get_latest_by(hostname=hostname) ]
 562                         else:
 563                                 nodes = FindbadNodeRecord.query.filter(FindbadNodeRecord.hostname.like(hostname))
 564
 565                         for node in nodes:
 566                                 lb = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 567                                 if lb not in loginbase_list:
 568                                         loginbase_list += [ lb ]
 569
 570                 if loginbase:
 571                         if not objtype:
 572                                 loginbase_list = [ loginbase ]
 573                         else:
 574                                 loginbase_list = HistorySiteRecord.query.filter(HistorySiteRecord.loginbase.like(loginbase))
 575                                 loginbase_list = [ l.loginbase for l in loginbase_list ]
 576
 577
 578                 if loginbase_list:
 579                         for loginbase in loginbase_list:
 580                                 actions = ActionRecord.query.filter_by(loginbase=loginbase
 581                                                                 ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since)
 582                                                                 ).order_by(ActionRecord.date_created.desc())
 583                                 actions_list += [ a for a in actions ]
 584                                 site = HistorySiteRecord.by_loginbase(loginbase)
 585                                 if site:
 586                                         sitequery.append(site)
 587                                 # NOTE: because a single pcu may be assigned to multiple hosts,
 588                                 # track unique pcus by their plc_pcuid, then turn dict into list
 589                                 pcus = {}
 590                                 for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 591                                                 # NOTE: reformat some fields.
 592                                                 agg = prep_node_for_display(node)
 593                                                 nodequery += [agg]
 594                                                 if agg.pcu:
 595                                                         pcus[agg.pcu.pcu.plc_pcuid] = agg.pcu
 596
 597                                 for pcuid_key in pcus:
 598                                         pcuquery += [pcus[pcuid_key]]
 599
 600                 actionlist_widget = ActionListWidget(template='monitorweb.templates.actionlist_template')
 601                 return dict(sitequery=sitequery, pcuquery=pcuquery, nodequery=nodequery, actions=actions_list, actionlist_widget=actionlist_widget, since=since, exceptions=exceptions)
 602
 603
 604         # TODO: add form validation
 605         @expose(template="monitorweb.templates.pcuview")
 606         @exception_handler(nodeaction_handler,"isinstance(tg_exceptions,RuntimeError)")
 607         def pcuviewold(self, loginbase=None, pcuid=None, hostname=None, since=20, **data):
 608                 session.flush(); session.clear()
 609                 sitequery=[]
 610                 pcuquery=[]
 611                 nodequery=[]
 612                 actions=[]
 613                 exceptions = None
 614
 615                 try: since = int(since)
 616                 except: since = 7
 617
 618                 for key in data:
 619                         print key, data[key]
 620
 621                 if 'submit' in data.keys() or 'type' in data.keys():
 622                         if hostname: data['hostname'] = hostname
 623                         self.nodeaction(**data)
 624                 if 'exceptions' in data:
 625                         exceptions = data['exceptions']
 626
 627                 if 'query' in data:
 628                         obj = data['query']
 629                         if len(obj.split(".")) > 1: hostname = obj
 630                         else: loginbase=obj
 631
 632                 if pcuid:
 633                         print "pcuid: %s" % pcuid
 634                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=pcuid)
 635                         loginbase = PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base']
 636
 637                 if hostname:
 638                         node = FindbadNodeRecord.get_latest_by(hostname=hostname)
 639                         loginbase = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 640
 641                 if loginbase:
 642                         actions = ActionRecord.query.filter_by(loginbase=loginbase
 643                                                         ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since)
 644                                                         ).order_by(ActionRecord.date_created.desc())
 645                         actions = [ a for a in actions ]
 646                         sitequery = [HistorySiteRecord.by_loginbase(loginbase)]
 647                         pcus = {}
 648                         for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 649                                         # NOTE: reformat some fields.
 650                                         agg = prep_node_for_display(node)
 651                                         nodequery += [agg]
 652                                         if agg.pcu: #.pcu.plc_pcuid:    # not None
 653                                                 #pcu = FindbadPCURecord.get_latest_by(plc_pcuid=agg.plc_pcuid)
 654                                                 #prep_pcu_for_display(pcu)
 655                                                 pcus[agg.pcu.pcu.plc_pcuid] = agg.pcu
 656
 657                         for pcuid_key in pcus:
 658                                 pcuquery += [pcus[pcuid_key]]
 659
 660                 return dict(sitequery=sitequery, pcuquery=pcuquery, nodequery=nodequery, actions=actions, since=since, exceptions=exceptions)
 661
 662         @expose(template="monitorweb.templates.pcuhistory")
 663         def pcuhistory(self, pcu_id=None):
 664                 query = []
 665                 if pcu_id:
 666                         fbnode = HistoryPCURecord.get_by(plc_pcuid=pcu_id)
 667                         l = fbnode.versions[-100:]
 668                         l.reverse()
 669                         for pcu in l:
 670                                 #prep_node_for_display(node)
 671                                 query.append(pcu)
 672
 673                 return dict(query=query, pcu_id=pcu_id)
 674
 675         @expose(template="monitorweb.templates.nodescanhistory")
 676         def nodescanhistory(self, hostname=None, length=10):
 677                 try: length = int(length)
 678                 except: length = 21
 679
 680                 fbnode = FindbadNodeRecord.get_by(hostname=hostname)
 681                 # TODO: add links for earlier history if desired.
 682                 l = fbnode.versions[-length:]
 683                 l.reverse()
 684                 query=[]
 685                 for node in l:
 686                         agg = prep_node_for_display(node, pcuhash=None, preppcu=False, asofdate=node.timestamp)
 687                         query.append(agg)
 688
 689                 if 'length' in request.params:
 690                         del request.params['length']
 691                 return dict(query=query, hostname=hostname, params=request.params)
 692
 693         @expose(template="monitorweb.templates.nodehistory")
 694         def nodehistory(self, hostname=None):
 695                 query = []
 696                 if hostname:
 697                         fbnode = HistoryNodeRecord.get_by(hostname=hostname)
 698                         l = fbnode.versions[-100:]
 699                         l.reverse()
 700                         for node in l:
 701                                 #prep_node_for_display(node)
 702                                 query.append(node)
 703
 704                 return dict(query=query, hostname=hostname)
 705
 706         @expose(template="monitorweb.templates.sitehistory")
 707         def sitehistory(self, loginbase=None):
 708                 query = []
 709                 if loginbase:
 710                         fbsite = HistorySiteRecord.get_by(loginbase=loginbase)
 711                         # TODO: add links for earlier history if desired.
 712                         l = fbsite.versions[-100:]
 713                         l.reverse()
 714                         for site in l:
 715                                 query.append(site)
 716                 return dict(query=query, loginbase=loginbase)
 717
 718
 719         @expose(template="monitorweb.templates.pculist")
 720         def pcu(self, filter='all'):
 721                 print "PCUVIEW------------------"
 722                 print "befor-len: ", len( [ i for i in session] )
 723                 session.flush(); session.clear()
 724                 print "after-len: ", len( [ i for i in session] )
 725                 fbquery = FindbadPCURecord.get_all_latest()
 726                 query = []
 727                 filtercount = {'ok' : 0, 'NetDown': 0, 'Not_Run' : 0, 'pending' : 0, 'all' : 0}
 728                 for node in fbquery:
 729
 730                         # NOTE: count filter
 731                         if node.reboot_trial_status == str(0):
 732                                 filtercount['ok'] += 1
 733                         elif node.reboot_trial_status == 'NetDown' or node.reboot_trial_status == 'Not_Run':
 734                                 filtercount[node.reboot_trial_status] += 1
 735                         else:
 736                                 filtercount['pending'] += 1
 737
 738                         pcuagg = prep_pcu_for_display(node)
 739
 740                         # NOTE: apply filter
 741                         if filter == "all":
 742                                 query.append(pcuagg)
 743                         elif filter == "ok" and node.reboot_trial_status == str(0):
 744                                 query.append(pcuagg)
 745                         elif filter == node.reboot_trial_status:
 746                                 query.append(pcuagg)
 747                         elif filter == "pending":
 748                                 # TODO: look in message logs...
 749                                 if node.reboot_trial_status != str(0) and \
 750                                         node.reboot_trial_status != 'NetDown' and \
 751                                         node.reboot_trial_status != 'Not_Run':
 752
 753                                         query.append(pcuagg)
 754
 755                 return dict(query=query, fc=filtercount)
 756
 757         @expose(template="monitorweb.templates.sitelist")
 758         def site(self, filter='all'):
 759                 print "SITE------------------"
 760                 print "befor-len: ", len( [ i for i in session] )
 761                 session.flush(); session.clear()
 762                 print "after-len: ", len( [ i for i in session] )
 763                 filtercount = {'good' : 0, 'down': 0, 'online':0, 'offline' : 0, 'new' : 0, 'pending' : 0, 'all' : 0}
 764                 fbquery = HistorySiteRecord.query.all()
 765                 query = []
 766                 for site in fbquery:
 767                         # count filter
 768                         filtercount['all'] += 1
 769                         if site.new and site.slices_used == 0 and not site.enabled:
 770                                 filtercount['new'] += 1
 771                         elif not site.enabled:
 772                                 filtercount['pending'] += 1
 773                         elif site.status in ['good', 'online']:
 774                                 filtercount['good'] += 1
 775                         elif site.status in ['down', 'offline']:
 776                                 filtercount['down'] += 1
 777
 778                         # apply filter
 779                         if filter == "all":
 780                                 query.append(site)
 781                         elif filter == 'new' and site.new and site.slices_used == 0 and not site.enabled:
 782                                 query.append(site)
 783                         elif filter == "pending" and not site.enabled:
 784                                 query.append(site)
 785                         elif filter == 'good' and site.status in ['good', 'online']:
 786                                 query.append(site)
 787                         elif filter == 'down' and site.status in ['down', 'offline']:
 788                                 query.append(site)
 789
 790                 return dict(query=query, fc=filtercount)
 791         @expose(template="monitorweb.templates.sitesummary")
 792         def sitesummary(self, loginbase="princeton"):
 793                 nodequery = []
 794                 for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 795                         agg = prep_node_for_display(node)
 796                         nodequery += [agg]
 797
 798                 return dict(nodequery=nodequery, loginbase=loginbase)
 799
 800         @expose(template="monitorweb.templates.summary")
 801         def summary(self, since=7):
 802                 sumdata = {}
 803                 sumdata['nodes'] = {}
 804                 sumdata['sites'] = {}
 805                 sumdata['pcus'] = {}
 806
 807                 def summarize(query, type):
 808                         for o in query:
 809                                 if o.status not in sumdata[type]:
 810                                         sumdata[type][o.status] = 0
 811                                 sumdata[type][o.status] += 1
 812
 813                 fbquery = HistorySiteRecord.query.all()
 814                 summarize(fbquery, 'sites')
 815                 fbquery = HistoryPCURecord.query.all()
 816                 summarize(fbquery, 'pcus')
 817                 fbquery = HistoryNodeRecord.query.all()
 818                 summarize(fbquery, 'nodes')
 819
 820                 if 'monitordebug' in sumdata['nodes']:
 821                         d = sumdata['nodes']['monitordebug']
 822                         del sumdata['nodes']['monitordebug']
 823                         sumdata['nodes']['failboot'] = d
 824
 825                 return dict(sumdata=sumdata, setorder=['good', 'offline', 'down', 'online'])
 826
 827         @expose(template="monitorweb.templates.actionsummary")
 828         def actionsummary(self, since=7):
 829                 from monitor.wrapper.emailTxt import mailtxt
 830
 831                 types = filter(lambda x: 'notice' in x, dir(mailtxt))
 832                 results = {}
 833
 834                 print mon_metadata.bind
 835                 if session.bind is None:
 836                         #TODO: figure out why this value gets cleared out...
 837                         session.bind = mon_metadata.bind
 838                 result = session.execute("select distinct(action_type) from actionrecord;")
 839
 840                 types = [r[0] for r in result]
 841
 842                 try: since = int(since)
 843                 except: since = 7
 844
 845                 for  t in types:
 846                         acts = ActionRecord.query.filter(ActionRecord.action_type==t
 847                                         ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since))
 848                         results[t] = acts.count()
 849                 return dict(results=results)
 850
 851         @expose(template="monitorweb.templates.actionlist")
 852         def actionlist(self, since=7, action_type=None, loginbase=None):
 853
 854                 try: since = int(since)
 855                 except: since = 7
 856
 857                 acts_query = ActionRecord.query.filter(
 858                                           ActionRecord.date_created >= datetime.now() - timedelta(since)
 859                                          )
 860                 if loginbase:
 861                         acts_query = acts_query.filter_by(loginbase=loginbase)
 862
 863                 if action_type:
 864                         acts_query = acts_query.filter(ActionRecord.action_type==action_type)
 865
 866                 acts = acts_query.order_by(ActionRecord.date_created.desc())
 867
 868                 query = [ a for a in acts ]
 869
 870                 return dict(actions=query, action_type=action_type, since=since)
 871
 872         @cherrypy.expose()
 873         def upload(self, log, **keywords):
 874                 hostname = None
 875                 logtype = None
 876                 logtype_list = ['bm.log', ]
 877
 878                 if 'hostname' in keywords:
 879                         hostname = keywords['hostname']
 880                 if 'type' in keywords and keywords['type'] in logtype_list:
 881                         logtype = keywords['type']
 882
 883                 if not hostname: return ""
 884                 if not logtype: return "unknown logtype: %s" % logtype
 885
 886                 short_target_filename = bootman.bootmanager_log_name(hostname)
 887                 abs_target_filename = os.path.join(config.MONITOR_BOOTMANAGER_LOG, short_target_filename)
 888                 print "write data: %s" % abs_target_filename
 889                 util.file.dumpFile(abs_target_filename, log.file.read())
 890                 bootman.bootmanager_log_action(hostname, short_target_filename, logtype)
 891                 session.flush()
 892
 893                 print "redirecting 3"
 894
 895                 return dict()