web/MonitorWeb/monitorweb/controllers.py

   1 import turbogears as tg
   2 from turbogears import controllers, expose, flash, exception_handler, redirect
   3 from turbogears import widgets
   4 from cherrypy import request, response
   5 import cherrypy
   6 # from monitorweb import model
   7 # import logging
   8 # log = logging.getLogger("monitorweb.controllers")
   9 import re
  10 import os
  11 from monitor.database.info.model import *
  12 #from monitor.database.zabbixapi.model import *
  13 from monitor_xmlrpc import MonitorXmlrpcServer
  14 from controllers_local import LocalExtensions
  15 from pcucontrol.reboot import pcu_name
  16
  17 from monitor import util
  18 from monitor import reboot
  19 from monitor import bootman
  20 from monitor import scanapi
  21 from monitor import config
  22 import time
  23
  24 from monitor.wrapper.plccache import plcdb_hn2lb as site_hn2lb
  25
  26 from monitorweb.templates.links import *
  27
  28 class ObjectQueryFields(widgets.WidgetsList):
  29         """The WidgetsList defines the fields of the form."""
  30         pass
  31
  32
  33
  34 class NodeQueryFields(widgets.WidgetsList):
  35         """The WidgetsList defines the fields of the form."""
  36
  37         object = widgets.RadioButtonList(label="Query Type", options=[('nodes', 'All Nodes'),
  38                                                                                                                           ('nodehistory', 'Single Node History'),
  39                                                                                                                           #('sites', 'All Sites'),
  40                                                                                                                           #('sitehistory', 'Single Site History'),
  41                                                                                                                           ], default="nodes")
  42         nodehistory_hostname = widgets.TextField(label="Hostname Node History", attrs={'size':30})
  43
  44         hostname = widgets.CheckBox(label="Hostname")
  45         firewall = widgets.CheckBox(label="Firewall?")
  46         fs_status = widgets.CheckBox(label="Filesystem Status")
  47         ssh_status = widgets.CheckBox(label="SSH Status")
  48         ssh_error = widgets.CheckBox(label="SSH Errors")
  49         dns_status = widgets.CheckBox(label="DNS Status")
  50         iptables_status = widgets.CheckBox(label="IP Tables Status")
  51         nm_status = widgets.CheckBox(label="NM Status")
  52         princeton_comon_dir = widgets.CheckBox(label="CoMon Dir")
  53         princeton_comon_running = widgets.CheckBox(label="CoMon Running")
  54         princeton_comon_procs = widgets.CheckBox(label="CoMon Processes")
  55         external_dns_status = widgets.CheckBox(label="Hostname Resolves?")
  56         kernel_version = widgets.CheckBox(label="Kernel")
  57         bootcd_version = widgets.CheckBox(label="BootCD")
  58         boot_server = widgets.CheckBox(label="Boot Server")
  59         install_date = widgets.CheckBox(label="Installation Date")
  60         observed_status = widgets.CheckBox(label="Observed Status")
  61         uptime = widgets.CheckBox(label="Uptime")
  62         traceroute = widgets.CheckBox(label="Traceroute")
  63         port_status = widgets.CheckBox(label="Port Status")
  64         plc_pcuid = widgets.CheckBox(label="PCU ID")
  65         rpms = widgets.CheckBox(label="RPM")
  66         rpmvalue = widgets.TextField(label="RPM Pattern")
  67
  68 class QueryForm(widgets.TableForm):
  69     template = """
  70     <form xmlns:py="http://purl.org/kid/ns#"
  71         id="queryform"
  72         name="${name}"
  73         action="${action}"
  74         method="${method}"
  75         class="tableform"
  76         py:attrs="form_attrs"
  77     >
  78         <div py:for="field in hidden_fields"
  79             py:replace="field.display(value_for(field), **params_for(field))"
  80         />
  81         <table border="0" cellspacing="0" cellpadding="2" py:attrs="table_attrs">
  82             <tr py:for="i, field in enumerate(fields)"
  83                 class="${i%2 and 'odd' or 'even'}"
  84             >
  85                 <th>
  86                     <label class="fieldlabel" for="${field.field_id}" py:content="field.label" />
  87                 </th>
  88                 <td>
  89                     <span py:replace="field.display(value_for(field), **params_for(field))" />
  90                     <span py:if="error_for(field)" class="fielderror" py:content="error_for(field)" />
  91                     <span py:if="field.help_text" class="fieldhelp" py:content="field.help_text" />
  92                 </td>
  93             </tr>
  94             <tr>
  95                 <td>&#160;</td>
  96                 <td py:content="submit.display(submit_text)" />
  97             </tr>
  98         </table>
  99     </form>
 100         """
 101
 102 def getNodeQueryForm():
 103         return QueryForm(fields=NodeQueryFields(), action="query")
 104
 105 # make it easier group objects without invoking the elixir auto-write feature.
 106 class aggregate: pass
 107
 108
 109 def query_to_dict(query):
 110         """ take a url query string and chop it up """
 111         val = {}
 112         query_fields = query.split('&')
 113         for f in query_fields:
 114                 (k,v) = urllib.splitvalue(f)
 115                 val[k] = v
 116
 117         return val
 118
 119 def format_ports(data, pcumodel=None):
 120         retval = []
 121         filtered_length=0
 122
 123         if pcumodel:
 124                 supported_ports=reboot.model_to_object(pcumodel).supported_ports
 125         else:
 126                 # ports of a production node
 127                 supported_ports=[22,80,806]
 128
 129         if data and len(data.keys()) > 0 :
 130                 for port in supported_ports:
 131                         try:
 132                                 state = data[str(port)]
 133                         except:
 134                                 state = "unknown"
 135
 136                         if state == "filtered":
 137                                 filtered_length += 1
 138
 139                         retval.append( (port, state) )
 140
 141         if retval == []:
 142                 retval = [( "Closed/Filtered", "state" )]
 143
 144         if filtered_length == len(supported_ports):
 145                 retval = [( "All Filtered", "state" )]
 146
 147         return retval
 148
 149 def format_pcu_shortstatus(pcu):
 150         status = "error"
 151         if pcu:
 152                 if pcu.reboot_trial_status == str(0) or pcu.reboot_trial_status == "Test: No error":
 153                         status = "Ok"
 154                 elif pcu.reboot_trial_status == "NetDown" or pcu.reboot_trial_status == "Not_Run":
 155                         status = pcu.reboot_trial_status
 156                 else:
 157                         status = "error"
 158
 159         return status
 160
 161 def prep_pcu_for_display(pcu):
 162         agg = aggregate()
 163         agg.pcu = pcu
 164
 165         try:
 166                 agg.loginbase = PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base']
 167         except:
 168                 agg.loginbase = "unknown"
 169
 170         agg.pcuhist = HistoryPCURecord.query.get(pcu.plc_pcuid)
 171
 172         agg.ports = format_ports(pcu.port_status, pcu.plc_pcu_stats['model'])
 173         agg.status = format_pcu_shortstatus(pcu)
 174         agg.pcu_name = pcu_name(pcu.plc_pcu_stats)
 175
 176         #print pcu.entry_complete
 177         agg.entry_complete_str = pcu.entry_complete
 178         #pcu.entry_complete_str += "".join([ f[0] for f in pcu.entry_complete.split() ])
 179         if pcu.dns_status == "NOHOSTNAME":
 180                 agg.dns_short_status = 'NoHost'
 181         elif pcu.dns_status == "DNS-OK":
 182                 agg.dns_short_status = 'Ok'
 183         elif pcu.dns_status == "DNS-NOENTRY":
 184                 agg.dns_short_status = 'NoEntry'
 185         elif pcu.dns_status == "NO-DNS-OR-IP":
 186                 agg.dns_short_status = 'NoHostOrIP'
 187         elif pcu.dns_status == "DNS-MISMATCH":
 188                 agg.dns_short_status = 'Mismatch'
 189         return agg
 190
 191 class ActionListWidget(widgets.Widget):
 192         pass
 193
 194 class NodeWidget(widgets.Widget):
 195         pass
 196
 197 def prep_nodehist(node):
 198         agg = aggregate()
 199         agg.node = node
 200         agg.loginbase = "unknown"
 201         try:
 202                 agg.loginbase = PlcSite.query.get(node.plc_siteid).plc_site_stats['login_base']
 203         except:
 204                 agg.loginbase = "exception"
 205
 206
 207         return agg
 208
 209 def prep_node_for_display(node, pcuhash=None, preppcu=True, asofdate=None):
 210         agg = aggregate()
 211         agg.node = node
 212
 213         if node.plc_pcuid and preppcu:
 214                 if pcuhash:
 215                         pcu = pcuhash[node.plc_pcuid]
 216                 else:
 217                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=node.plc_pcuid)
 218
 219                 if pcu:
 220                         agg.pcu_status = pcu.reboot_trial_status
 221                         agg.pcu_short_status = format_pcu_shortstatus(pcu)
 222                         agg.pcu = prep_pcu_for_display(pcu)
 223                 else:
 224                         agg.pcu_short_status = "none"
 225                         agg.pcu_status = "nodata"
 226                         agg.pcu = None
 227
 228         else:
 229                 agg.pcu_status = "nopcu"
 230                 agg.pcu_short_status = "none"
 231                 agg.pcu = None
 232
 233
 234         if node.kernel_version:
 235                 agg.kernel = node.kernel_version.split()[2]
 236         else:
 237                 agg.kernel = ""
 238
 239         try:
 240                 agg.loginbase = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 241         except:
 242                 agg.loginbase = "unknown"
 243
 244         if agg.loginbase:
 245                 agg.site = HistorySiteRecord.by_loginbase(agg.loginbase)
 246
 247                 if asofdate:
 248                         agg.site = agg.site.get_as_of(asofdate)
 249
 250                 if agg.site is None:
 251                         # TODO: need a cleaner fix for this...
 252                         agg.site = HistorySiteRecord.by_loginbase("pl")
 253                         if not agg.site:
 254                                 agg.site = HistorySiteRecord.by_loginbase("ple")
 255
 256         agg.history = HistoryNodeRecord.by_hostname(node.hostname)
 257         if asofdate:
 258                 agg.history = agg.history.get_as_of(asofdate)
 259
 260         agg.ports = format_ports(node.port_status)
 261
 262         try:
 263                 exists = node.plc_node_stats['last_contact']
 264         except:
 265                 # TODO: this should not assign to the fb object!
 266                 node.plc_node_stats = {'last_contact' : None}
 267
 268         return agg
 269
 270
 271 class Root(controllers.RootController, MonitorXmlrpcServer, LocalExtensions):
 272         @expose(template="monitorweb.templates.welcome")
 273         def index(self):
 274                 # log.debug("Happy TurboGears Controller Responding For Duty")
 275                 flash("Welcome To MyOps!")
 276                 return dict(now=time.ctime())
 277
 278         @expose(template="monitorweb.templates.nodelist", allow_json=True)
 279         def node3(self, filter=None):
 280                 nhquery = HistoryNodeRecord.query.all()
 281                 query = []
 282                 for nh in nhquery:
 283                         if filter:
 284                                 if nh.status == filter:
 285                                         query.append(nh)
 286                         else:
 287                                 query.append(nh)
 288
 289                 rquery=[]
 290                 for q in query:
 291                         fb = FindbadNodeRecord.get_latest_by(hostname=q.hostname)
 292                         rquery.append(fb)
 293
 294                 return dict(now=time.ctime(), query=rquery)
 295
 296         def node_query(self, filter):
 297                 nhquery = HistoryNodeRecord.query.all()
 298                 query = []
 299                 for nh in nhquery:
 300                         if filter:
 301                                 if nh.status == filter:
 302                                         query.append(nh)
 303                         else:
 304                                 query.append(nh)
 305
 306                 rquery=[]
 307                 for q in query:
 308                         fb = FindbadNodeRecord.get_latest_by(hostname=q.hostname)
 309                         agg = prep_node_for_display(fb)
 310                         rquery.append(agg)
 311                 return rquery
 312
 313         @expose("cheetah:monitorweb.templates.nodelist_plain", as_format="plain",
 314                 accept_format="text/plain", content_type="text/plain")
 315         @expose(template="monitorweb.templates.nodelist", allow_json=True)
 316         def node2(self, filter=None):
 317                 rquery=self.node_query(filter)
 318                 widget = NodeWidget(template='monitorweb.templates.node_template')
 319                 return dict(now=time.ctime(), query=rquery, nodewidget=widget)
 320
 321         @expose("cheetah:monitorweb.templates.query_plain", as_format="plain",
 322                 accept_format="text/plain", content_type="text/plain")
 323         @expose(template="monitorweb.templates.query", allow_json=True)
 324         def query(self, **data):
 325                 query = []
 326
 327                 for k in data:
 328                         print k, data[k]
 329
 330                 fbquery = None
 331
 332                 if 'object' in data and data['object'] == "nodes":
 333                         fbquery = FindbadNodeRecord.get_all_latest()
 334                 elif 'object' in data and data['object'] == "nodehistory":
 335                         hostname = data['nodehistory_hostname']
 336                         data['date_checked'] = 'date_checked'
 337                         fbrecord = FindbadNodeRecord.get_by(hostname=hostname)
 338                         fbquery = fbrecord.versions[-500:]
 339
 340                 if fbquery:
 341                         for node in fbquery:
 342                                 # NOTE: reformat some fields.
 343                                 if type(node) is not type(FindbadNodeRecord):
 344                                         agg = node.__dict__.copy()
 345                                 else:
 346                                         agg = node.to_dict()
 347                                 if agg['plc_node_stats']:
 348                                         agg.update(agg['plc_node_stats'])
 349                                 if agg['install_date']:
 350                                         agg['install_date'] = time.mktime(time.strptime(agg['install_date'], "%a %b %d %H:%M:%S %Y"))
 351                                 if agg['kernel_version']:
 352                                         agg['kernel_version'] = agg['kernel_version'].split()[2]
 353                                 if 'traceroute' in data and agg['traceroute']:
 354                                         agg['traceroute'] = "<pre>" + agg['traceroute'] + "</pre>"
 355                                 if 'rpmvalue' in data and 'rpms' in data:
 356                                         if agg['rpms']:
 357                                                 rpm_list = agg['rpms'].split()
 358                                                 rpm_list = filter(lambda x: re.match(data['rpmvalue'], x, re.I),
 359                                                                   rpm_list)
 360                                                 agg['rpms'] = " ".join(rpm_list)
 361
 362                                 query.append(agg)
 363
 364                 fields=data.copy()
 365
 366                 try:
 367                         del fields['object']
 368                         del fields['rpmvalue']
 369                         del fields['nodehistory_hostname']
 370                 except: pass
 371                 return dict(now=time.ctime(), query=query, fields=fields, data=data, queryform=getNodeQueryForm())
 372
 373         @expose(template="monitorweb.templates.nodefast", allow_json=True)
 374         def node(self, filter=None):
 375                 nhquery = HistoryNodeRecord.query.all()
 376                 query = []
 377                 for nh in nhquery:
 378                         if filter:
 379                                 if nh.status == filter:
 380                                         agg = prep_nodehist(nh)
 381                                         query.append(agg)
 382                         else:
 383                                 agg = prep_nodehist(nh)
 384                                 query.append(agg)
 385
 386                 return dict(now=time.ctime(), query=query)
 387
 388         @expose(template="monitorweb.templates.nodelist")
 389         def nodeslow(self, filter='boot'):
 390                 print "NODE------------------"
 391                 print "befor-len: ", len( [ i for i in session] )
 392                 session.flush(); session.clear()
 393                 print "after-len: ", len( [ i for i in session] )
 394                 fbquery = FindbadNodeRecord.get_all_latest()
 395                 query = []
 396                 filtercount = {'down' : 0, 'boot': 0, 'debug' : 0, 'diagnose' : 0, 'disabled': 0,
 397                                                 'neverboot' : 0, 'pending' : 0, 'all' : 0, None : 0}
 398                 for node in fbquery:
 399                         # NOTE: reformat some fields.
 400                         agg = prep_node_for_display(node)
 401
 402                         if not agg.history:
 403                                 continue
 404
 405                         if agg.history.status in ['down', 'offline']:
 406                                 if node.plc_node_stats and node.plc_node_stats['last_contact'] != None:
 407                                         filtercount['down'] += 1
 408                                 else:
 409                                         filtercount['neverboot'] += 1
 410                         elif agg.history.status in ['good', 'online']:
 411                                 filtercount['boot'] += 1
 412                         elif agg.history.status in ['debug', 'monitordebug']:
 413                                 filtercount['debug'] += 1
 414                         else:
 415                                 if filtercount.has_key(agg.history.status):
 416                                         filtercount[agg.history.status] += 1
 417
 418
 419                         # NOTE: apply filter
 420                         if filter == "neverboot":
 421                                 if not node.plc_node_stats or node.plc_node_stats['last_contact'] == None:
 422                                         query.append(agg)
 423                         elif filter == "all":
 424                                 query.append(agg)
 425                         elif filter == agg.history.status:
 426                                 query.append(agg)
 427                         elif filter == 'boot':
 428                                 query.append(agg)
 429
 430
 431                 widget = NodeWidget(template='monitorweb.templates.node_template')
 432                 return dict(now=time.ctime(), query=query, fc=filtercount, nodewidget=widget)
 433
 434         def nodeaction_handler(self, tg_exceptions=None):
 435                 """Handle any kind of error."""
 436                 print "NODEACTION_HANDLER------------------"
 437
 438                 if 'pcuid' in request.params:
 439                         pcuid = request.params['pcuid']
 440                 else:
 441                         refurl = request.headers.get("Referer",link("pcu"))
 442                         print refurl
 443
 444                         # TODO: do this more intelligently...
 445                         uri_fields = urllib.splitquery(refurl)
 446                         if uri_fields[1] is not None:
 447                                 val = query_to_dict(uri_fields[1])
 448                                 if 'pcuid' in val:
 449                                         pcuid = val['pcuid']
 450                                 elif 'hostname' in val:
 451                                         pcuid = FindbadNodeRecord.get_latest_by(hostname=val['hostname']).plc_pcuid
 452                                 else:
 453                                         pcuid=None
 454                         else:
 455                                 pcuid=None
 456
 457                 cherry_trail = cherrypy._cputil.get_object_trail()
 458                 for i in cherry_trail:
 459                         print "trail: ", i
 460
 461                 print pcuid
 462                 return self.pcuview(None, pcuid, **dict(exceptions=tg_exceptions))
 463
 464         def nodeaction(self, **data):
 465                 print "NODEACTION------------------"
 466                 for item in data.keys():
 467                         print "%s %s" % ( item, data[item] )
 468
 469                 if 'hostname' in data:
 470                         hostname = data['hostname']
 471                 else:
 472                         flash("No hostname given in submitted data")
 473                         return
 474
 475                 if 'submit' in data or 'type' in data:
 476                         try:
 477                                 action = data['submit']
 478                         except:
 479                                 action = data['type']
 480                 else:
 481                         flash("No submit action given in submitted data")
 482                         return
 483
 484                 if action == "Reboot":
 485                         print "REBOOT: %s" % hostname
 486                         ret = reboot.reboot_str(str(hostname))
 487                         print ret
 488                         if ret: raise RuntimeError("Error using PCU: " + str(ret))
 489                         flash("Reboot appeared to work.  Allow at most 5 minutes.  Then run ExternalScan to check current status.")
 490
 491                 elif action == "ExternalScan":
 492                         scanapi.externalprobe(str(hostname))
 493                         flash("External Scan Successful!")
 494                 elif action == "InternalScan":
 495                         scanapi.internalprobe(str(hostname))
 496                         flash("Internal Scan Successful!")
 497                 else:
 498                         # unknown action
 499                         raise RuntimeError("Unknown action given")
 500                 return
 501
 502         @expose(template="monitorweb.templates.simpleview")
 503         def simpleview(self, **data):
 504                 return self.pre_view(**data)
 505
 506         @expose(template="monitorweb.templates.simpleview")
 507         def pcuview(self, **data):
 508                 return self.pre_view(**data)
 509
 510         @expose(template="monitorweb.templates.detailview")
 511         def detailview(self, **data):
 512                 return self.pre_view(**data)
 513
 514
 515         def pre_view(self, **data):
 516                 session.flush(); session.clear()
 517
 518                 loginbase=None
 519                 loginbase_list=[]
 520                 hostname=None
 521                 pcuid=None
 522                 since=20
 523                 # if objtype is not None, then treat 'hostname' or 'loginbase' as a search pattern
 524                 objtype=None
 525
 526                 exceptions = None
 527                 sitequery=[]
 528                 nodequery=[]
 529                 pcuquery=[]
 530                 actions=[]
 531                 actions_list=[]
 532
 533                 for key in data:
 534                         print key, data[key]
 535
 536                 if 'query' in data:
 537                         obj = data['query']
 538                         fields = obj.split(":")
 539                         if len(fields) > 1:
 540                                 objtype = fields[0]
 541                                 obj = fields[1].replace("*", "%")
 542                                 print "obj: %s"% obj
 543
 544                         if len(obj.split(".")) > 1 or objtype == "node":
 545                                 hostname = obj
 546                         else:
 547                                 loginbase = obj
 548
 549                 if 'loginbase' in data:
 550                         loginbase = data['loginbase']
 551
 552                 if 'hostname' in data:
 553                         hostname = data['hostname']
 554
 555                 if 'pcuid' in data:
 556                         try: pcuid = int(data['pcuid'])
 557                         except: pcuid = None
 558
 559                 if 'since' in data:
 560                         try: since = int(since)
 561                         except: since = 20
 562
 563                 if pcuid:
 564                         print "pcuid: %s" % pcuid
 565                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=pcuid)
 566                         loginbase_list += [ PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base'] ]
 567
 568                 if hostname:
 569                         if not objtype:
 570                                 nodes = [ FindbadNodeRecord.get_latest_by(hostname=hostname) ]
 571                         else:
 572                                 nodes = FindbadNodeRecord.query.filter(FindbadNodeRecord.hostname.like(hostname))
 573
 574                         for node in nodes:
 575                                 lb = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 576                                 if lb not in loginbase_list:
 577                                         loginbase_list += [ lb ]
 578
 579                 if loginbase:
 580                         if not objtype:
 581                                 loginbase_list = [ loginbase ]
 582                         else:
 583                                 loginbase_list = HistorySiteRecord.query.filter(HistorySiteRecord.loginbase.like(loginbase))
 584                                 loginbase_list = [ l.loginbase for l in loginbase_list ]
 585
 586
 587                 if loginbase_list:
 588                         for loginbase in loginbase_list:
 589                                 actions = ActionRecord.query.filter_by(loginbase=loginbase
 590                                                                 ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since)
 591                                                                 ).order_by(ActionRecord.date_created.desc())
 592                                 actions_list += [ a for a in actions ]
 593                                 site = HistorySiteRecord.by_loginbase(loginbase)
 594                                 if site:
 595                                         sitequery.append(site)
 596                                 # NOTE: because a single pcu may be assigned to multiple hosts,
 597                                 # track unique pcus by their plc_pcuid, then turn dict into list
 598                                 pcus = {}
 599                                 for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 600                                                 # NOTE: reformat some fields.
 601                                                 agg = prep_node_for_display(node)
 602                                                 nodequery += [agg]
 603                                                 if agg.pcu:
 604                                                         pcus[agg.pcu.pcu.plc_pcuid] = agg.pcu
 605
 606                                 for pcuid_key in pcus:
 607                                         pcuquery += [pcus[pcuid_key]]
 608
 609                 #for a in nodequery:
 610                 #       print type(a.node)
 611                 #       print type(a.node.hostname)
 612                 nodequery.sort(lambda a,b: cmp(a.node.hostname,b.node.hostname))
 613                 pcuquery.sort(lambda a,b: cmp(a.pcu_name,b.pcu_name))
 614
 615                 actionlist_widget = ActionListWidget(template='monitorweb.templates.actionlist_template')
 616                 return dict(sitequery=sitequery, pcuquery=pcuquery, nodequery=nodequery, actions=actions_list, actionlist_widget=actionlist_widget, since=since, exceptions=exceptions)
 617
 618
 619         # TODO: add form validation
 620         @expose(template="monitorweb.templates.pcuview")
 621         @exception_handler(nodeaction_handler,"isinstance(tg_exceptions,RuntimeError)")
 622         def pcuviewold(self, loginbase=None, pcuid=None, hostname=None, since=20, **data):
 623                 session.flush(); session.clear()
 624                 sitequery=[]
 625                 pcuquery=[]
 626                 nodequery=[]
 627                 actions=[]
 628                 exceptions = None
 629
 630                 try: since = int(since)
 631                 except: since = 7
 632
 633                 for key in data:
 634                         print key, data[key]
 635
 636                 if 'submit' in data.keys() or 'type' in data.keys():
 637                         if hostname: data['hostname'] = hostname
 638                         self.nodeaction(**data)
 639                 if 'exceptions' in data:
 640                         exceptions = data['exceptions']
 641
 642                 if 'query' in data:
 643                         obj = data['query']
 644                         if len(obj.split(".")) > 1: hostname = obj
 645                         else: loginbase=obj
 646
 647                 if pcuid:
 648                         print "pcuid: %s" % pcuid
 649                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=pcuid)
 650                         loginbase = PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base']
 651
 652                 if hostname:
 653                         node = FindbadNodeRecord.get_latest_by(hostname=hostname)
 654                         loginbase = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 655
 656                 if loginbase:
 657                         actions = ActionRecord.query.filter_by(loginbase=loginbase
 658                                                         ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since)
 659                                                         ).order_by(ActionRecord.date_created.desc())
 660                         actions = [ a for a in actions ]
 661                         sitequery = [HistorySiteRecord.by_loginbase(loginbase)]
 662                         pcus = {}
 663                         for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 664                                         # NOTE: reformat some fields.
 665                                         agg = prep_node_for_display(node)
 666                                         nodequery += [agg]
 667                                         if agg.pcu: #.pcu.plc_pcuid:    # not None
 668                                                 #pcu = FindbadPCURecord.get_latest_by(plc_pcuid=agg.plc_pcuid)
 669                                                 #prep_pcu_for_display(pcu)
 670                                                 pcus[agg.pcu.pcu.plc_pcuid] = agg.pcu
 671
 672                         for pcuid_key in pcus:
 673                                 pcuquery += [pcus[pcuid_key]]
 674
 675                 return dict(sitequery=sitequery, pcuquery=pcuquery, nodequery=nodequery, actions=actions, since=since, exceptions=exceptions)
 676
 677         @expose(template="monitorweb.templates.pcuhistory")
 678         def pcuhistory(self, pcu_id=None):
 679                 query = []
 680                 if pcu_id:
 681                         fbnode = HistoryPCURecord.get_by(plc_pcuid=pcu_id)
 682                         l = fbnode.versions[-1000:]
 683                         l.reverse()
 684                         for pcu in l:
 685                                 #prep_node_for_display(node)
 686                                 query.append(pcu)
 687
 688                 return dict(query=query, pcu_id=pcu_id)
 689
 690         @expose(template="monitorweb.templates.nodescanhistory")
 691         def nodescanhistory(self, hostname=None, length=10):
 692                 try: length = int(length)
 693                 except: length = 21
 694
 695                 fbnode = FindbadNodeRecord.get_by(hostname=hostname)
 696                 # TODO: add links for earlier history if desired.
 697                 l = fbnode.versions[-length:]
 698                 l.reverse()
 699                 query=[]
 700                 for node in l:
 701                         agg = prep_node_for_display(node, pcuhash=None, preppcu=False, asofdate=node.timestamp)
 702                         query.append(agg)
 703
 704                 if 'length' in request.params:
 705                         del request.params['length']
 706                 return dict(query=query, hostname=hostname, params=request.params)
 707
 708         @expose(template="monitorweb.templates.nodehistory")
 709         def nodehistory(self, hostname=None):
 710                 query = []
 711                 if hostname:
 712                         fbnode = HistoryNodeRecord.get_by(hostname=hostname)
 713                         l = fbnode.versions[-100:]
 714                         l.reverse()
 715                         for node in l:
 716                                 #prep_node_for_display(node)
 717                                 query.append(node)
 718
 719                 return dict(query=query, hostname=hostname)
 720
 721         @expose(template="monitorweb.templates.sitehistory")
 722         def sitehistory(self, loginbase=None):
 723                 query = []
 724                 if loginbase:
 725                         fbsite = HistorySiteRecord.get_by(loginbase=loginbase)
 726                         # TODO: add links for earlier history if desired.
 727                         l = fbsite.versions[-1000:]
 728                         l.reverse()
 729                         for site in l:
 730                                 query.append(site)
 731                 return dict(query=query, loginbase=loginbase)
 732
 733
 734         @expose("cheetah:monitorweb.templates.pculist_plain", as_format="plain",
 735                 accept_format="text/plain", content_type="text/plain")
 736         @expose(template="monitorweb.templates.pculist")
 737         def pcu(self, filter='all'):
 738                 print "PCUVIEW------------------"
 739                 print "befor-len: ", len( [ i for i in session] )
 740                 session.flush(); session.clear()
 741                 print "after-len: ", len( [ i for i in session] )
 742                 fbquery = FindbadPCURecord.get_all_latest()
 743                 query = []
 744                 filtercount = {'ok' : 0, 'NetDown': 0, 'Not_Run' : 0, 'pending' : 0, 'all' : 0}
 745                 for node in fbquery:
 746
 747                         # NOTE: count filter
 748                         if node.reboot_trial_status == str(0):
 749                                 filtercount['ok'] += 1
 750                         elif node.reboot_trial_status == 'NetDown' or node.reboot_trial_status == 'Not_Run':
 751                                 filtercount[node.reboot_trial_status] += 1
 752                         else:
 753                                 filtercount['pending'] += 1
 754
 755                         pcuagg = prep_pcu_for_display(node)
 756
 757                         # NOTE: apply filter
 758                         if filter == "all":
 759                                 query.append(pcuagg)
 760                         elif filter == "ok" and node.reboot_trial_status == str(0):
 761                                 query.append(pcuagg)
 762                         elif filter == node.reboot_trial_status:
 763                                 query.append(pcuagg)
 764                         elif filter == "pending":
 765                                 # TODO: look in message logs...
 766                                 if node.reboot_trial_status != str(0) and \
 767                                         node.reboot_trial_status != 'NetDown' and \
 768                                         node.reboot_trial_status != 'Not_Run':
 769
 770                                         query.append(pcuagg)
 771
 772                 return dict(query=query, fc=filtercount)
 773
 774         @expose(template="monitorweb.templates.sitelist")
 775         def site(self, filter='all'):
 776                 print "SITE------------------"
 777                 print "befor-len: ", len( [ i for i in session] )
 778                 session.flush(); session.clear()
 779                 print "after-len: ", len( [ i for i in session] )
 780                 filtercount = {'good' : 0, 'down': 0, 'online':0, 'offline' : 0, 'new' : 0, 'pending' : 0, 'all' : 0}
 781                 fbquery = HistorySiteRecord.query.all()
 782                 query = []
 783                 for site in fbquery:
 784                         # count filter
 785                         filtercount['all'] += 1
 786                         if site.new and site.slices_used == 0 and not site.enabled:
 787                                 filtercount['new'] += 1
 788                         elif not site.enabled:
 789                                 filtercount['pending'] += 1
 790                         elif site.status in ['good', 'online']:
 791                                 filtercount['good'] += 1
 792                         elif site.status in ['down', 'offline']:
 793                                 filtercount['down'] += 1
 794
 795                         # apply filter
 796                         if filter == "all":
 797                                 query.append(site)
 798                         elif filter == 'new' and site.new and site.slices_used == 0 and not site.enabled:
 799                                 query.append(site)
 800                         elif filter == "pending" and not site.enabled:
 801                                 query.append(site)
 802                         elif filter == 'good' and site.status in ['good', 'online']:
 803                                 query.append(site)
 804                         elif filter == 'down' and site.status in ['down', 'offline']:
 805                                 query.append(site)
 806
 807                 return dict(query=query, fc=filtercount)
 808         @expose(template="monitorweb.templates.sitesummary")
 809         def sitesummary(self, loginbase="princeton"):
 810                 nodequery = []
 811                 for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 812                         agg = prep_node_for_display(node)
 813                         nodequery += [agg]
 814
 815                 return dict(nodequery=nodequery, loginbase=loginbase)
 816
 817         @expose(template="monitorweb.templates.summary")
 818         def summary(self, since=7):
 819                 sumdata = {}
 820                 sumdata['nodes'] = {}
 821                 sumdata['sites'] = {}
 822                 sumdata['pcus'] = {}
 823
 824                 def summarize(query, type):
 825                         for o in query:
 826                                 if o.status not in sumdata[type]:
 827                                         sumdata[type][o.status] = 0
 828                                 sumdata[type][o.status] += 1
 829
 830                 fbquery = HistorySiteRecord.query.all()
 831                 summarize(fbquery, 'sites')
 832                 fbquery = HistoryPCURecord.query.all()
 833                 summarize(fbquery, 'pcus')
 834                 fbquery = HistoryNodeRecord.query.all()
 835                 summarize(fbquery, 'nodes')
 836
 837                 if 'monitordebug' in sumdata['nodes']:
 838                         d = sumdata['nodes']['monitordebug']
 839                         del sumdata['nodes']['monitordebug']
 840                         sumdata['nodes']['failboot'] = d
 841
 842                 return dict(sumdata=sumdata, setorder=['good', 'offline', 'down', 'online'])
 843
 844         @expose(template="monitorweb.templates.actionsummary")
 845         def actionsummary(self, since=7):
 846                 from monitor.wrapper.emailTxt import mailtxt
 847
 848                 types = filter(lambda x: 'notice' in x, dir(mailtxt))
 849                 results = {}
 850
 851                 print mon_metadata.bind
 852                 if session.bind is None:
 853                         #TODO: figure out why this value gets cleared out...
 854                         session.bind = mon_metadata.bind
 855                 result = session.execute("select distinct(action_type) from actionrecord;")
 856
 857                 types = [r[0] for r in result]
 858
 859                 try: since = int(since)
 860                 except: since = 7
 861
 862                 for  t in types:
 863                         acts = ActionRecord.query.filter(ActionRecord.action_type==t
 864                                         ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since))
 865                         results[t] = acts.count()
 866                 return dict(results=results)
 867
 868         @expose(template="monitorweb.templates.actionlist")
 869         def actionlist(self, since=7, action_type=None, loginbase=None):
 870
 871                 try: since = int(since)
 872                 except: since = 7
 873
 874                 acts_query = ActionRecord.query.filter(
 875                                           ActionRecord.date_created >= datetime.now() - timedelta(since)
 876                                          )
 877                 if loginbase:
 878                         acts_query = acts_query.filter_by(loginbase=loginbase)
 879
 880                 if action_type:
 881                         acts_query = acts_query.filter(ActionRecord.action_type==action_type)
 882
 883                 acts = acts_query.order_by(ActionRecord.date_created.desc())
 884
 885                 query = [ a for a in acts ]
 886
 887                 return dict(actions=query, action_type=action_type, since=since)
 888
 889         @cherrypy.expose()
 890         def upload(self, log, **keywords):
 891                 hostname = None
 892                 logtype = None
 893                 logtype_list = ['bm.log', ]
 894
 895                 if 'hostname' in keywords:
 896                         hostname = keywords['hostname']
 897                 if 'type' in keywords and keywords['type'] in logtype_list:
 898                         logtype = keywords['type']
 899
 900                 if not hostname: return ""
 901                 if not logtype: return "unknown logtype: %s" % logtype
 902
 903                 short_target_filename = bootman.bootmanager_log_name(hostname)
 904                 abs_target_filename = os.path.join(config.MONITOR_BOOTMANAGER_LOG, short_target_filename)
 905                 print "write data: %s" % abs_target_filename
 906                 util.file.dumpFile(abs_target_filename, log.file.read())
 907                 bootman.bootmanager_log_action(hostname, short_target_filename, logtype)
 908                 session.flush()
 909
 910                 print "redirecting 3"
 911
 912                 return dict()
 913