web/MonitorWeb/monitorweb/controllers.py

   1 import turbogears as tg
   2 from turbogears import controllers, expose, flash, exception_handler, redirect
   3 from turbogears import widgets
   4 from cherrypy import request, response
   5 import cherrypy
   6 # from monitorweb import model
   7 # import logging
   8 # log = logging.getLogger("monitorweb.controllers")
   9 import re
  10 import os
  11 from monitor.database.info.model import *
  12 #from monitor.database.zabbixapi.model import *
  13 from monitor_xmlrpc import MonitorXmlrpcServer
  14
  15 from monitor import reboot
  16 from monitor import scanapi
  17 import time
  18
  19 from monitor.wrapper.plccache import plcdb_hn2lb as site_hn2lb
  20
  21 from monitorweb.templates.links import *
  22
  23 class ObjectQueryFields(widgets.WidgetsList):
  24         """The WidgetsList defines the fields of the form."""
  25         pass
  26
  27
  28
  29 class NodeQueryFields(widgets.WidgetsList):
  30         """The WidgetsList defines the fields of the form."""
  31
  32         object = widgets.RadioButtonList(label="Query Type", options=[('nodes', 'All Nodes'),
  33                                                                                                                           ('nodehistory', 'Single Node History'),
  34                                                                                                                           #('sites', 'All Sites'),
  35                                                                                                                           #('sitehistory', 'Single Site History'),
  36                                                                                                                           ], default="nodes")
  37         nodehistory_hostname = widgets.TextField(label="Hostname Node History", attrs={'size':30})
  38
  39         hostname = widgets.CheckBox(label="Hostname")
  40         firewall = widgets.CheckBox(label="Firewall?")
  41         ssh_status = widgets.CheckBox(label="SSH Status")
  42         ssh_error = widgets.CheckBox(label="SSH Errors")
  43         dns_status = widgets.CheckBox(label="DNS Status")
  44         nm_status = widgets.CheckBox(label="NM Status")
  45         princeton_comon_dir = widgets.CheckBox(label="CoMon Dir")
  46         princeton_comon_running = widgets.CheckBox(label="CoMon Running")
  47         princeton_comon_procs = widgets.CheckBox(label="CoMon Processes")
  48         external_dns_status = widgets.CheckBox(label="Hostname Resolves?")
  49         kernel_version = widgets.CheckBox(label="Kernel")
  50         bootcd_version = widgets.CheckBox(label="BootCD")
  51         observed_status = widgets.CheckBox(label="Observed Status")
  52         port_status = widgets.CheckBox(label="Port Status")
  53         rpms = widgets.CheckBox(label="RPM")
  54         rpmvalue = widgets.TextField(label="RPM Pattern")
  55
  56 class QueryForm(widgets.TableForm):
  57     template = """
  58     <form xmlns:py="http://purl.org/kid/ns#"
  59         name="${name}"
  60         action="${action}"
  61         method="${method}"
  62         class="tableform"
  63         py:attrs="form_attrs"
  64     >
  65         <div py:for="field in hidden_fields"
  66             py:replace="field.display(value_for(field), **params_for(field))"
  67         />
  68         <table border="0" cellspacing="0" cellpadding="2" py:attrs="table_attrs">
  69             <tr py:for="i, field in enumerate(fields)"
  70                 class="${i%2 and 'odd' or 'even'}"
  71             >
  72                 <th>
  73                     <label class="fieldlabel" for="${field.field_id}" py:content="field.label" />
  74                 </th>
  75                 <td>
  76                     <span py:replace="field.display(value_for(field), **params_for(field))" />
  77                     <span py:if="error_for(field)" class="fielderror" py:content="error_for(field)" />
  78                     <span py:if="field.help_text" class="fieldhelp" py:content="field.help_text" />
  79                 </td>
  80             </tr>
  81             <tr>
  82                 <td>&#160;</td>
  83                 <td py:content="submit.display(submit_text)" />
  84             </tr>
  85         </table>
  86     </form>
  87         """
  88
  89 def getNodeQueryForm():
  90         return QueryForm(fields=NodeQueryFields(), action="query")
  91
  92 # make it easier group objects without invoking the elixir auto-write feature.
  93 class aggregate: pass
  94
  95
  96 def query_to_dict(query):
  97         """ take a url query string and chop it up """
  98         val = {}
  99         query_fields = query.split('&')
 100         for f in query_fields:
 101                 (k,v) = urllib.splitvalue(f)
 102                 val[k] = v
 103
 104         return val
 105
 106 def format_ports(data, pcumodel=None):
 107         retval = []
 108         filtered_length=0
 109
 110         if pcumodel:
 111                 supported_ports=reboot.model_to_object(pcumodel).supported_ports
 112         else:
 113                 # ports of a production node
 114                 supported_ports=[22,80,806]
 115
 116         if data and len(data.keys()) > 0 :
 117                 for port in supported_ports:
 118                         try:
 119                                 state = data[str(port)]
 120                         except:
 121                                 state = "unknown"
 122
 123                         if state == "filtered":
 124                                 filtered_length += 1
 125
 126                         retval.append( (port, state) )
 127
 128         if retval == []:
 129                 retval = [( "Closed/Filtered", "state" )]
 130
 131         if filtered_length == len(supported_ports):
 132                 retval = [( "All Filtered", "state" )]
 133
 134         return retval
 135
 136 def format_pcu_shortstatus(pcu):
 137         status = "error"
 138         if pcu:
 139                 if pcu.reboot_trial_status == str(0):
 140                         status = "Ok"
 141                 elif pcu.reboot_trial_status == "NetDown" or pcu.reboot_trial_status == "Not_Run":
 142                         status = pcu.reboot_trial_status
 143                 else:
 144                         status = "error"
 145
 146         return status
 147
 148 def prep_pcu_for_display(pcu):
 149         agg = aggregate()
 150         agg.pcu = pcu
 151
 152         try:
 153                 agg.loginbase = PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base']
 154         except:
 155                 agg.loginbase = "unknown"
 156
 157         agg.pcuhist = HistoryPCURecord.query.get(pcu.plc_pcuid)
 158
 159         agg.ports = format_ports(pcu.port_status, pcu.plc_pcu_stats['model'])
 160         agg.status = format_pcu_shortstatus(pcu)
 161
 162         #print pcu.entry_complete
 163         agg.entry_complete_str = pcu.entry_complete
 164         #pcu.entry_complete_str += "".join([ f[0] for f in pcu.entry_complete.split() ])
 165         if pcu.dns_status == "NOHOSTNAME":
 166                 agg.dns_short_status = 'NoHost'
 167         elif pcu.dns_status == "DNS-OK":
 168                 agg.dns_short_status = 'Ok'
 169         elif pcu.dns_status == "DNS-NOENTRY":
 170                 agg.dns_short_status = 'NoEntry'
 171         elif pcu.dns_status == "NO-DNS-OR-IP":
 172                 agg.dns_short_status = 'NoHostOrIP'
 173         elif pcu.dns_status == "DNS-MISMATCH":
 174                 agg.dns_short_status = 'Mismatch'
 175         return agg
 176
 177 class NodeWidget(widgets.Widget):
 178         pass
 179
 180 def prep_nodehist(node):
 181         agg = aggregate()
 182         agg.node = node
 183         agg.loginbase = "unknown"
 184         try:
 185                 agg.loginbase = PlcSite.query.get(node.plc_siteid).plc_site_stats['login_base']
 186         except:
 187                 agg.loginbase = "exception"
 188
 189
 190         return agg
 191
 192 def prep_node_for_display(node, pcuhash=None, preppcu=True, asofdate=None):
 193         agg = aggregate()
 194         agg.node = node
 195
 196         if node.plc_pcuid and preppcu:
 197                 if pcuhash:
 198                         pcu = pcuhash[node.plc_pcuid]
 199                 else:
 200                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=node.plc_pcuid)
 201
 202                 if pcu:
 203                         agg.pcu_status = pcu.reboot_trial_status
 204                         agg.pcu_short_status = format_pcu_shortstatus(pcu)
 205                         agg.pcu = prep_pcu_for_display(pcu)
 206                 else:
 207                         agg.pcu_short_status = "none"
 208                         agg.pcu_status = "nodata"
 209                         agg.pcu = None
 210
 211         else:
 212                 agg.pcu_status = "nopcu"
 213                 agg.pcu_short_status = "none"
 214                 agg.pcu = None
 215
 216
 217         if node.kernel_version:
 218                 agg.kernel = node.kernel_version.split()[2]
 219         else:
 220                 agg.kernel = ""
 221
 222         try:
 223                 agg.loginbase = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 224         except:
 225                 agg.loginbase = "unknown"
 226
 227         if agg.loginbase:
 228                 agg.site = HistorySiteRecord.by_loginbase(agg.loginbase)
 229
 230                 if asofdate:
 231                         agg.site = agg.site.get_as_of(asofdate)
 232
 233                 if agg.site is None:
 234                         # TODO: need a cleaner fix for this...
 235                         agg.site = HistorySiteRecord.by_loginbase("pl")
 236                         if not agg.site:
 237                                 agg.site = HistorySiteRecord.by_loginbase("ple")
 238
 239         agg.history = HistoryNodeRecord.by_hostname(node.hostname)
 240         if asofdate:
 241                 agg.history = agg.history.get_as_of(asofdate)
 242
 243         agg.ports = format_ports(node.port_status)
 244
 245         try:
 246                 exists = node.plc_node_stats['last_contact']
 247         except:
 248                 # TODO: this should not assign to the fb object!
 249                 node.plc_node_stats = {'last_contact' : None}
 250
 251         return agg
 252
 253
 254 class Root(controllers.RootController, MonitorXmlrpcServer):
 255         @expose(template="monitorweb.templates.welcome")
 256         def index(self):
 257                 # log.debug("Happy TurboGears Controller Responding For Duty")
 258                 flash("Welcome To MyOps!")
 259                 return dict(now=time.ctime())
 260
 261         @expose(template="monitorweb.templates.nodelist", allow_json=True)
 262         def node3(self, filter=None):
 263                 nhquery = HistoryNodeRecord.query.all()
 264                 query = []
 265                 for nh in nhquery:
 266                         if filter:
 267                                 if nh.status == filter:
 268                                         query.append(nh)
 269                         else:
 270                                 query.append(nh)
 271
 272                 rquery=[]
 273                 for q in query:
 274                         fb = FindbadNodeRecord.get_latest_by(hostname=q.hostname)
 275                         rquery.append(fb)
 276
 277                 return dict(now=time.ctime(), query=rquery)
 278
 279         def node_query(self, filter):
 280                 nhquery = HistoryNodeRecord.query.all()
 281                 query = []
 282                 for nh in nhquery:
 283                         if filter:
 284                                 if nh.status == filter:
 285                                         query.append(nh)
 286                         else:
 287                                 query.append(nh)
 288
 289                 rquery=[]
 290                 for q in query:
 291                         fb = FindbadNodeRecord.get_latest_by(hostname=q.hostname)
 292                         agg = prep_node_for_display(fb)
 293                         rquery.append(agg)
 294                 return rquery
 295
 296         @expose("cheetah:monitorweb.templates.nodelist_plain", as_format="plain",
 297                 accept_format="text/plain", content_type="text/plain")
 298         @expose(template="monitorweb.templates.nodelist", allow_json=True)
 299         def node2(self, filter=None):
 300                 rquery=self.node_query(filter)
 301                 widget = NodeWidget(template='monitorweb.templates.node_template')
 302                 return dict(now=time.ctime(), query=rquery, nodewidget=widget)
 303
 304         @expose("cheetah:monitorweb.templates.query_plain", as_format="plain",
 305                 accept_format="text/plain", content_type="text/plain")
 306         @expose(template="monitorweb.templates.query", allow_json=True)
 307         def query(self, **data):
 308                 query = []
 309
 310                 for k in data:
 311                         print k, data[k]
 312
 313                 fbquery = None
 314
 315                 if 'object' in data and data['object'] == "nodes":
 316                         fbquery = FindbadNodeRecord.get_all_latest()
 317                 elif 'object' in data and data['object'] == "nodehistory":
 318                         hostname = data['nodehistory_hostname']
 319                         data['date_checked'] = 'date_checked'
 320                         fbrecord = FindbadNodeRecord.get_by(hostname=hostname)
 321                         fbquery = fbrecord.versions[-500:]
 322
 323                 if fbquery:
 324                         for node in fbquery:
 325                                 # NOTE: reformat some fields.
 326                                 if type(node) is not type(FindbadNodeRecord):
 327                                         agg = node.__dict__.copy()
 328                                 else:
 329                                         agg = node.to_dict()
 330                                 agg.update(agg['plc_node_stats'])
 331                                 if agg['kernel_version']:
 332                                         agg['kernel_version'] = agg['kernel_version'].split()[2]
 333                                 if 'rpmvalue' in data and 'rpms' in data:
 334                                         if agg['rpms']:
 335                                                 rpm_list = agg['rpms'].split()
 336                                                 rpm_list = filter(lambda x: data['rpmvalue'] in x, rpm_list)
 337                                                 agg['rpms'] = " ".join(rpm_list)
 338
 339                                 query.append(agg)
 340
 341                 fields=data.copy()
 342
 343                 try:
 344                         del fields['object']
 345                         del fields['rpmvalue']
 346                         del fields['nodehistory_hostname']
 347                 except: pass
 348                 return dict(now=time.ctime(), query=query, fields=fields, data=data, queryform=getNodeQueryForm())
 349
 350         @expose(template="monitorweb.templates.nodefast", allow_json=True)
 351         def node(self, filter=None):
 352                 nhquery = HistoryNodeRecord.query.all()
 353                 query = []
 354                 for nh in nhquery:
 355                         if filter:
 356                                 if nh.status == filter:
 357                                         agg = prep_nodehist(nh)
 358                                         query.append(agg)
 359                         else:
 360                                 agg = prep_nodehist(nh)
 361                                 query.append(agg)
 362
 363                 return dict(now=time.ctime(), query=query)
 364
 365         @expose(template="monitorweb.templates.nodelist")
 366         def nodeslow(self, filter='boot'):
 367                 print "NODE------------------"
 368                 print "befor-len: ", len( [ i for i in session] )
 369                 session.flush(); session.clear()
 370                 print "after-len: ", len( [ i for i in session] )
 371                 fbquery = FindbadNodeRecord.get_all_latest()
 372                 query = []
 373                 filtercount = {'down' : 0, 'boot': 0, 'debug' : 0, 'diagnose' : 0, 'disabled': 0,
 374                                                 'neverboot' : 0, 'pending' : 0, 'all' : 0, None : 0}
 375                 for node in fbquery:
 376                         # NOTE: reformat some fields.
 377                         agg = prep_node_for_display(node)
 378
 379                         if not agg.history:
 380                                 continue
 381
 382                         if agg.history.status in ['down', 'offline']:
 383                                 if node.plc_node_stats and node.plc_node_stats['last_contact'] != None:
 384                                         filtercount['down'] += 1
 385                                 else:
 386                                         filtercount['neverboot'] += 1
 387                         elif agg.history.status in ['good', 'online']:
 388                                 filtercount['boot'] += 1
 389                         elif agg.history.status in ['debug', 'monitordebug']:
 390                                 filtercount['debug'] += 1
 391                         else:
 392                                 if filtercount.has_key(agg.history.status):
 393                                         filtercount[agg.history.status] += 1
 394
 395
 396                         # NOTE: apply filter
 397                         if filter == "neverboot":
 398                                 if not node.plc_node_stats or node.plc_node_stats['last_contact'] == None:
 399                                         query.append(agg)
 400                         elif filter == "all":
 401                                 query.append(agg)
 402                         elif filter == agg.history.status:
 403                                 query.append(agg)
 404                         elif filter == 'boot':
 405                                 query.append(agg)
 406
 407
 408                 widget = NodeWidget(template='monitorweb.templates.node_template')
 409                 return dict(now=time.ctime(), query=query, fc=filtercount, nodewidget=widget)
 410
 411         def nodeaction_handler(self, tg_exceptions=None):
 412                 """Handle any kind of error."""
 413                 print "NODEACTION_HANDLER------------------"
 414
 415                 if 'pcuid' in request.params:
 416                         pcuid = request.params['pcuid']
 417                 else:
 418                         refurl = request.headers.get("Referer",link("pcu"))
 419                         print refurl
 420
 421                         # TODO: do this more intelligently...
 422                         uri_fields = urllib.splitquery(refurl)
 423                         if uri_fields[1] is not None:
 424                                 val = query_to_dict(uri_fields[1])
 425                                 if 'pcuid' in val:
 426                                         pcuid = val['pcuid']
 427                                 elif 'hostname' in val:
 428                                         pcuid = FindbadNodeRecord.get_latest_by(hostname=val['hostname']).plc_pcuid
 429                                 else:
 430                                         pcuid=None
 431                         else:
 432                                 pcuid=None
 433
 434                 cherry_trail = cherrypy._cputil.get_object_trail()
 435                 for i in cherry_trail:
 436                         print "trail: ", i
 437
 438                 print pcuid
 439                 return self.pcuview(None, pcuid, **dict(exceptions=tg_exceptions))
 440
 441         def nodeaction(self, **data):
 442                 print "NODEACTION------------------"
 443                 for item in data.keys():
 444                         print "%s %s" % ( item, data[item] )
 445
 446                 if 'hostname' in data:
 447                         hostname = data['hostname']
 448                 else:
 449                         flash("No hostname given in submitted data")
 450                         return
 451
 452                 if 'submit' in data or 'type' in data:
 453                         try:
 454                                 action = data['submit']
 455                         except:
 456                                 action = data['type']
 457                 else:
 458                         flash("No submit action given in submitted data")
 459                         return
 460
 461                 if action == "Reboot":
 462                         print "REBOOT: %s" % hostname
 463                         ret = reboot.reboot_str(str(hostname))
 464                         print ret
 465                         if ret: raise RuntimeError("Error using PCU: " + str(ret))
 466                         flash("Reboot appeared to work.  Allow at most 5 minutes.  Then run ExternalScan to check current status.")
 467
 468                 elif action == "ExternalScan":
 469                         scanapi.externalprobe(str(hostname))
 470                         flash("External Scan Successful!")
 471                 elif action == "InternalScan":
 472                         scanapi.internalprobe(str(hostname))
 473                         flash("Internal Scan Successful!")
 474                 else:
 475                         # unknown action
 476                         raise RuntimeError("Unknown action given")
 477                 return
 478
 479         @expose(template="monitorweb.templates.simpleview")
 480         def simpleview(self, **data):
 481                 return self.pre_view(**data)
 482
 483         @expose(template="monitorweb.templates.detailview")
 484         def detailview(self, **data):
 485                 return self.pre_view(**data)
 486
 487         def pre_view(self, **data):
 488                 session.flush(); session.clear()
 489
 490                 loginbase=None
 491                 loginbase_list=[]
 492                 hostname=None
 493                 pcuid=None
 494                 since=20
 495                 # if objtype is not None, then treat 'hostname' or 'loginbase' as a search pattern
 496                 objtype=None
 497
 498                 exceptions = None
 499                 sitequery=[]
 500                 nodequery=[]
 501                 pcuquery=[]
 502                 actions=[]
 503                 actions_list=[]
 504
 505                 for key in data:
 506                         print key, data[key]
 507
 508                 if 'query' in data:
 509                         obj = data['query']
 510                         fields = obj.split(":")
 511                         if len(fields) > 1:
 512                                 objtype = fields[0]
 513                                 obj = fields[1].replace("*", "%")
 514                                 print "obj: %s"% obj
 515
 516                         if len(obj.split(".")) > 1 or objtype == "node":
 517                                 hostname = obj
 518                         else:
 519                                 loginbase = obj
 520
 521                 if 'loginbase' in data:
 522                         loginbase = data['loginbase']
 523
 524                 if 'hostname' in data:
 525                         hostname = data['hostname']
 526
 527                 if 'pcuid' in data:
 528                         try: pcuid = int(data['pcuid'])
 529                         except: pcuid = None
 530
 531                 if 'since' in data:
 532                         try: since = int(since)
 533                         except: since = 20
 534
 535                 if pcuid:
 536                         print "pcuid: %s" % pcuid
 537                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=pcuid)
 538                         loginbase_list += [ PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base'] ]
 539
 540                 if hostname:
 541                         if not objtype:
 542                                 nodes = [ FindbadNodeRecord.get_latest_by(hostname=hostname) ]
 543                         else:
 544                                 nodes = FindbadNodeRecord.query.filter(FindbadNodeRecord.hostname.like(hostname))
 545
 546                         for node in nodes:
 547                                 lb = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 548                                 if lb not in loginbase_list:
 549                                         loginbase_list += [ lb ]
 550
 551                 if loginbase:
 552                         if not objtype:
 553                                 loginbase_list = [ loginbase ]
 554                         else:
 555                                 loginbase_list = HistorySiteRecord.query.filter(HistorySiteRecord.loginbase.like(loginbase))
 556                                 loginbase_list = [ l.loginbase for l in loginbase_list ]
 557
 558
 559                 if loginbase_list:
 560                         for loginbase in loginbase_list:
 561                                 actions = ActionRecord.query.filter_by(loginbase=loginbase
 562                                                                 ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since)
 563                                                                 ).order_by(ActionRecord.date_created.desc())
 564                                 actions_list += [ a for a in actions ]
 565                                 site = HistorySiteRecord.by_loginbase(loginbase)
 566                                 if site:
 567                                         sitequery.append(site)
 568                                 # NOTE: because a single pcu may be assigned to multiple hosts,
 569                                 # track unique pcus by their plc_pcuid, then turn dict into list
 570                                 pcus = {}
 571                                 for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 572                                                 # NOTE: reformat some fields.
 573                                                 agg = prep_node_for_display(node)
 574                                                 nodequery += [agg]
 575                                                 if agg.pcu:
 576                                                         pcus[agg.pcu.pcu.plc_pcuid] = agg.pcu
 577
 578                                 for pcuid_key in pcus:
 579                                         pcuquery += [pcus[pcuid_key]]
 580
 581                 return dict(sitequery=sitequery, pcuquery=pcuquery, nodequery=nodequery, actions=actions_list, since=since, exceptions=exceptions)
 582
 583
 584         # TODO: add form validation
 585         @expose(template="monitorweb.templates.pcuview")
 586         @exception_handler(nodeaction_handler,"isinstance(tg_exceptions,RuntimeError)")
 587         def pcuview(self, loginbase=None, pcuid=None, hostname=None, since=20, **data):
 588                 session.flush(); session.clear()
 589                 sitequery=[]
 590                 pcuquery=[]
 591                 nodequery=[]
 592                 actions=[]
 593                 exceptions = None
 594
 595                 try: since = int(since)
 596                 except: since = 7
 597
 598                 for key in data:
 599                         print key, data[key]
 600
 601                 if 'submit' in data.keys() or 'type' in data.keys():
 602                         if hostname: data['hostname'] = hostname
 603                         self.nodeaction(**data)
 604                 if 'exceptions' in data:
 605                         exceptions = data['exceptions']
 606
 607                 if 'query' in data:
 608                         obj = data['query']
 609                         if len(obj.split(".")) > 1: hostname = obj
 610                         else: loginbase=obj
 611
 612                 if pcuid:
 613                         print "pcuid: %s" % pcuid
 614                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=pcuid)
 615                         loginbase = PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base']
 616
 617                 if hostname:
 618                         node = FindbadNodeRecord.get_latest_by(hostname=hostname)
 619                         loginbase = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 620
 621                 if loginbase:
 622                         actions = ActionRecord.query.filter_by(loginbase=loginbase
 623                                                         ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since)
 624                                                         ).order_by(ActionRecord.date_created.desc())
 625                         actions = [ a for a in actions ]
 626                         sitequery = [HistorySiteRecord.by_loginbase(loginbase)]
 627                         pcus = {}
 628                         for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 629                                         # NOTE: reformat some fields.
 630                                         agg = prep_node_for_display(node)
 631                                         nodequery += [agg]
 632                                         if agg.pcu: #.pcu.plc_pcuid:    # not None
 633                                                 #pcu = FindbadPCURecord.get_latest_by(plc_pcuid=agg.plc_pcuid)
 634                                                 #prep_pcu_for_display(pcu)
 635                                                 pcus[agg.pcu.pcu.plc_pcuid] = agg.pcu
 636
 637                         for pcuid_key in pcus:
 638                                 pcuquery += [pcus[pcuid_key]]
 639
 640                 return dict(sitequery=sitequery, pcuquery=pcuquery, nodequery=nodequery, actions=actions, since=since, exceptions=exceptions)
 641
 642         @expose(template="monitorweb.templates.pcuhistory")
 643         def pcuhistory(self, pcu_id=None):
 644                 query = []
 645                 if pcu_id:
 646                         fbnode = HistoryPCURecord.get_by(plc_pcuid=pcu_id)
 647                         l = fbnode.versions[-100:]
 648                         l.reverse()
 649                         for pcu in l:
 650                                 #prep_node_for_display(node)
 651                                 query.append(pcu)
 652
 653                 return dict(query=query, pcu_id=pcu_id)
 654
 655         @expose(template="monitorweb.templates.nodescanhistory")
 656         def nodescanhistory(self, hostname=None, length=10):
 657                 try: length = int(length)
 658                 except: length = 21
 659
 660                 fbnode = FindbadNodeRecord.get_by(hostname=hostname)
 661                 # TODO: add links for earlier history if desired.
 662                 l = fbnode.versions[-length:]
 663                 l.reverse()
 664                 query=[]
 665                 for node in l:
 666                         agg = prep_node_for_display(node, pcuhash=None, preppcu=False, asofdate=node.timestamp)
 667                         query.append(agg)
 668
 669                 if 'length' in request.params:
 670                         del request.params['length']
 671                 return dict(query=query, hostname=hostname, params=request.params)
 672
 673         @expose(template="monitorweb.templates.nodehistory")
 674         def nodehistory(self, hostname=None):
 675                 query = []
 676                 if hostname:
 677                         fbnode = HistoryNodeRecord.get_by(hostname=hostname)
 678                         l = fbnode.versions[-100:]
 679                         l.reverse()
 680                         for node in l:
 681                                 #prep_node_for_display(node)
 682                                 query.append(node)
 683
 684                 return dict(query=query, hostname=hostname)
 685
 686         @expose(template="monitorweb.templates.sitehistory")
 687         def sitehistory(self, loginbase=None):
 688                 query = []
 689                 if loginbase:
 690                         fbsite = HistorySiteRecord.get_by(loginbase=loginbase)
 691                         # TODO: add links for earlier history if desired.
 692                         l = fbsite.versions[-100:]
 693                         l.reverse()
 694                         for site in l:
 695                                 query.append(site)
 696                 return dict(query=query, loginbase=loginbase)
 697
 698
 699         @expose(template="monitorweb.templates.pculist")
 700         def pcu(self, filter='all'):
 701                 print "PCUVIEW------------------"
 702                 print "befor-len: ", len( [ i for i in session] )
 703                 session.flush(); session.clear()
 704                 print "after-len: ", len( [ i for i in session] )
 705                 fbquery = FindbadPCURecord.get_all_latest()
 706                 query = []
 707                 filtercount = {'ok' : 0, 'NetDown': 0, 'Not_Run' : 0, 'pending' : 0, 'all' : 0}
 708                 for node in fbquery:
 709
 710                         # NOTE: count filter
 711                         if node.reboot_trial_status == str(0):
 712                                 filtercount['ok'] += 1
 713                         elif node.reboot_trial_status == 'NetDown' or node.reboot_trial_status == 'Not_Run':
 714                                 filtercount[node.reboot_trial_status] += 1
 715                         else:
 716                                 filtercount['pending'] += 1
 717
 718                         pcuagg = prep_pcu_for_display(node)
 719
 720                         # NOTE: apply filter
 721                         if filter == "all":
 722                                 query.append(pcuagg)
 723                         elif filter == "ok" and node.reboot_trial_status == str(0):
 724                                 query.append(pcuagg)
 725                         elif filter == node.reboot_trial_status:
 726                                 query.append(pcuagg)
 727                         elif filter == "pending":
 728                                 # TODO: look in message logs...
 729                                 if node.reboot_trial_status != str(0) and \
 730                                         node.reboot_trial_status != 'NetDown' and \
 731                                         node.reboot_trial_status != 'Not_Run':
 732
 733                                         query.append(pcuagg)
 734
 735                 return dict(query=query, fc=filtercount)
 736
 737         @expose(template="monitorweb.templates.sitelist")
 738         def site(self, filter='all'):
 739                 print "SITE------------------"
 740                 print "befor-len: ", len( [ i for i in session] )
 741                 session.flush(); session.clear()
 742                 print "after-len: ", len( [ i for i in session] )
 743                 filtercount = {'good' : 0, 'down': 0, 'online':0, 'offline' : 0, 'new' : 0, 'pending' : 0, 'all' : 0}
 744                 fbquery = HistorySiteRecord.query.all()
 745                 query = []
 746                 for site in fbquery:
 747                         # count filter
 748                         filtercount['all'] += 1
 749                         if site.new and site.slices_used == 0 and not site.enabled:
 750                                 filtercount['new'] += 1
 751                         elif not site.enabled:
 752                                 filtercount['pending'] += 1
 753                         elif site.status in ['good', 'online']:
 754                                 filtercount['good'] += 1
 755                         elif site.status in ['down', 'offline']:
 756                                 filtercount['down'] += 1
 757
 758                         # apply filter
 759                         if filter == "all":
 760                                 query.append(site)
 761                         elif filter == 'new' and site.new and site.slices_used == 0 and not site.enabled:
 762                                 query.append(site)
 763                         elif filter == "pending" and not site.enabled:
 764                                 query.append(site)
 765                         elif filter == 'good' and site.status in ['good', 'online']:
 766                                 query.append(site)
 767                         elif filter == 'down' and site.status in ['down', 'offline']:
 768                                 query.append(site)
 769
 770                 return dict(query=query, fc=filtercount)
 771         @expose(template="monitorweb.templates.sitesummary")
 772         def sitesummary(self, loginbase="princeton"):
 773                 nodequery = []
 774                 for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 775                         agg = prep_node_for_display(node)
 776                         nodequery += [agg]
 777
 778                 return dict(nodequery=nodequery, loginbase=loginbase)
 779
 780         @expose(template="monitorweb.templates.summary")
 781         def summary(self, since=7):
 782                 sumdata = {}
 783                 sumdata['nodes'] = {}
 784                 sumdata['sites'] = {}
 785                 sumdata['pcus'] = {}
 786
 787                 def summarize(query, type):
 788                         for o in query:
 789                                 if o.status not in sumdata[type]:
 790                                         sumdata[type][o.status] = 0
 791                                 sumdata[type][o.status] += 1
 792
 793                 fbquery = HistorySiteRecord.query.all()
 794                 summarize(fbquery, 'sites')
 795                 fbquery = HistoryPCURecord.query.all()
 796                 summarize(fbquery, 'pcus')
 797                 fbquery = HistoryNodeRecord.query.all()
 798                 summarize(fbquery, 'nodes')
 799
 800                 if 'monitordebug' in sumdata['nodes']:
 801                         d = sumdata['nodes']['monitordebug']
 802                         del sumdata['nodes']['monitordebug']
 803                         sumdata['nodes']['failboot'] = d
 804
 805                 return dict(sumdata=sumdata, setorder=['good', 'offline', 'down', 'online'])
 806
 807         @expose(template="monitorweb.templates.actionsummary")
 808         def actionsummary(self, since=7):
 809                 from monitor.wrapper.emailTxt import mailtxt
 810
 811                 types = filter(lambda x: 'notice' in x, dir(mailtxt))
 812                 results = {}
 813
 814                 print mon_metadata.bind
 815                 if session.bind is None:
 816                         #TODO: figure out why this value gets cleared out...
 817                         session.bind = mon_metadata.bind
 818                 result = session.execute("select distinct(action_type) from actionrecord;")
 819
 820                 types = [r[0] for r in result]
 821
 822                 try: since = int(since)
 823                 except: since = 7
 824
 825                 for  t in types:
 826                         acts = ActionRecord.query.filter(ActionRecord.action_type==t
 827                                         ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since))
 828                         results[t] = acts.count()
 829                 return dict(results=results)
 830
 831         @expose(template="monitorweb.templates.actionlist")
 832         def actionlist(self, action_type='down_notice', since=7, loginbase=None):
 833
 834                 try: since = int(since)
 835                 except: since = 7
 836
 837                 if loginbase:
 838                         acts = ActionRecord.query.filter_by(loginbase=loginbase
 839                                 ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since)
 840                                 ).order_by(ActionRecord.date_created.desc())
 841                 else:
 842                         acts = ActionRecord.query.filter(ActionRecord.action_type==action_type
 843                                 ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since)
 844                                 ).order_by(ActionRecord.date_created.desc())
 845                 query = [ a for a in acts ]
 846
 847                 return dict(actions=query, action_type=action_type, since=since)
 848
 849         @cherrypy.expose()
 850         def upload(self, log, **keywords):
 851                 print "got data"
 852                 data = log.file.read()
 853                 target_file_name = os.path.join(os.getcwd(), log.filename)
 854                 # open file in binary mode for writing
 855
 856                 f = open(target_file_name, 'wb')
 857                 print "write data"
 858                 f.write(data)
 859                 f.close()
 860
 861                 #flash("File uploaded successfully: %s saved as: %s" \
 862                 #               % (upload_file.filename, target_file_name))
 863                 #u = UploadedFile(filename=upload_file.filename,
 864                 #  abspath=target_file_name, size=0)
 865                 print "redirecting "
 866
 867                 #redirect("monitor")
 868                 return dict()