web/MonitorWeb/monitorweb/controllers.py

   1 import turbogears as tg
   2 from turbogears import controllers, expose, flash, exception_handler, redirect
   3 from turbogears import widgets
   4 from cherrypy import request, response
   5 import cherrypy
   6 # from monitorweb import model
   7 # import logging
   8 # log = logging.getLogger("monitorweb.controllers")
   9 import re
  10 import os
  11 from monitor.database.info.model import *
  12 #from monitor.database.zabbixapi.model import *
  13 from monitor_xmlrpc import MonitorXmlrpcServer
  14
  15 from monitor import util
  16 from monitor import reboot
  17 from monitor import bootman
  18 from monitor import scanapi
  19 from monitor import config
  20 import time
  21
  22 from monitor.wrapper.plccache import plcdb_hn2lb as site_hn2lb
  23
  24 from monitorweb.templates.links import *
  25
  26 class ObjectQueryFields(widgets.WidgetsList):
  27         """The WidgetsList defines the fields of the form."""
  28         pass
  29
  30
  31
  32 class NodeQueryFields(widgets.WidgetsList):
  33         """The WidgetsList defines the fields of the form."""
  34
  35         object = widgets.RadioButtonList(label="Query Type", options=[('nodes', 'All Nodes'),
  36                                                                                                                           ('nodehistory', 'Single Node History'),
  37                                                                                                                           #('sites', 'All Sites'),
  38                                                                                                                           #('sitehistory', 'Single Site History'),
  39                                                                                                                           ], default="nodes")
  40         nodehistory_hostname = widgets.TextField(label="Hostname Node History", attrs={'size':30})
  41
  42         hostname = widgets.CheckBox(label="Hostname")
  43         firewall = widgets.CheckBox(label="Firewall?")
  44         fs_status = widgets.CheckBox(label="Filesystem Status")
  45         ssh_status = widgets.CheckBox(label="SSH Status")
  46         ssh_error = widgets.CheckBox(label="SSH Errors")
  47         dns_status = widgets.CheckBox(label="DNS Status")
  48         iptables_status = widgets.CheckBox(label="IP Tables Status")
  49         nm_status = widgets.CheckBox(label="NM Status")
  50         princeton_comon_dir = widgets.CheckBox(label="CoMon Dir")
  51         princeton_comon_running = widgets.CheckBox(label="CoMon Running")
  52         princeton_comon_procs = widgets.CheckBox(label="CoMon Processes")
  53         external_dns_status = widgets.CheckBox(label="Hostname Resolves?")
  54         kernel_version = widgets.CheckBox(label="Kernel")
  55         bootcd_version = widgets.CheckBox(label="BootCD")
  56         observed_status = widgets.CheckBox(label="Observed Status")
  57         uptime = widgets.CheckBox(label="Uptime")
  58         port_status = widgets.CheckBox(label="Port Status")
  59         rpms = widgets.CheckBox(label="RPM")
  60         rpmvalue = widgets.TextField(label="RPM Pattern")
  61
  62 class QueryForm(widgets.TableForm):
  63     template = """
  64     <form xmlns:py="http://purl.org/kid/ns#"
  65         name="${name}"
  66         action="${action}"
  67         method="${method}"
  68         class="tableform"
  69         py:attrs="form_attrs"
  70     >
  71         <div py:for="field in hidden_fields"
  72             py:replace="field.display(value_for(field), **params_for(field))"
  73         />
  74         <table border="0" cellspacing="0" cellpadding="2" py:attrs="table_attrs">
  75             <tr py:for="i, field in enumerate(fields)"
  76                 class="${i%2 and 'odd' or 'even'}"
  77             >
  78                 <th>
  79                     <label class="fieldlabel" for="${field.field_id}" py:content="field.label" />
  80                 </th>
  81                 <td>
  82                     <span py:replace="field.display(value_for(field), **params_for(field))" />
  83                     <span py:if="error_for(field)" class="fielderror" py:content="error_for(field)" />
  84                     <span py:if="field.help_text" class="fieldhelp" py:content="field.help_text" />
  85                 </td>
  86             </tr>
  87             <tr>
  88                 <td>&#160;</td>
  89                 <td py:content="submit.display(submit_text)" />
  90             </tr>
  91         </table>
  92     </form>
  93         """
  94
  95 def getNodeQueryForm():
  96         return QueryForm(fields=NodeQueryFields(), action="query")
  97
  98 # make it easier group objects without invoking the elixir auto-write feature.
  99 class aggregate: pass
 100
 101
 102 def query_to_dict(query):
 103         """ take a url query string and chop it up """
 104         val = {}
 105         query_fields = query.split('&')
 106         for f in query_fields:
 107                 (k,v) = urllib.splitvalue(f)
 108                 val[k] = v
 109
 110         return val
 111
 112 def format_ports(data, pcumodel=None):
 113         retval = []
 114         filtered_length=0
 115
 116         if pcumodel:
 117                 supported_ports=reboot.model_to_object(pcumodel).supported_ports
 118         else:
 119                 # ports of a production node
 120                 supported_ports=[22,80,806]
 121
 122         if data and len(data.keys()) > 0 :
 123                 for port in supported_ports:
 124                         try:
 125                                 state = data[str(port)]
 126                         except:
 127                                 state = "unknown"
 128
 129                         if state == "filtered":
 130                                 filtered_length += 1
 131
 132                         retval.append( (port, state) )
 133
 134         if retval == []:
 135                 retval = [( "Closed/Filtered", "state" )]
 136
 137         if filtered_length == len(supported_ports):
 138                 retval = [( "All Filtered", "state" )]
 139
 140         return retval
 141
 142 def format_pcu_shortstatus(pcu):
 143         status = "error"
 144         if pcu:
 145                 if pcu.reboot_trial_status == str(0):
 146                         status = "Ok"
 147                 elif pcu.reboot_trial_status == "NetDown" or pcu.reboot_trial_status == "Not_Run":
 148                         status = pcu.reboot_trial_status
 149                 else:
 150                         status = "error"
 151
 152         return status
 153
 154 def prep_pcu_for_display(pcu):
 155         agg = aggregate()
 156         agg.pcu = pcu
 157
 158         try:
 159                 agg.loginbase = PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base']
 160         except:
 161                 agg.loginbase = "unknown"
 162
 163         agg.pcuhist = HistoryPCURecord.query.get(pcu.plc_pcuid)
 164
 165         agg.ports = format_ports(pcu.port_status, pcu.plc_pcu_stats['model'])
 166         agg.status = format_pcu_shortstatus(pcu)
 167
 168         #print pcu.entry_complete
 169         agg.entry_complete_str = pcu.entry_complete
 170         #pcu.entry_complete_str += "".join([ f[0] for f in pcu.entry_complete.split() ])
 171         if pcu.dns_status == "NOHOSTNAME":
 172                 agg.dns_short_status = 'NoHost'
 173         elif pcu.dns_status == "DNS-OK":
 174                 agg.dns_short_status = 'Ok'
 175         elif pcu.dns_status == "DNS-NOENTRY":
 176                 agg.dns_short_status = 'NoEntry'
 177         elif pcu.dns_status == "NO-DNS-OR-IP":
 178                 agg.dns_short_status = 'NoHostOrIP'
 179         elif pcu.dns_status == "DNS-MISMATCH":
 180                 agg.dns_short_status = 'Mismatch'
 181         return agg
 182
 183 class ActionListWidget(widgets.Widget):
 184         pass
 185
 186 class NodeWidget(widgets.Widget):
 187         pass
 188
 189 def prep_nodehist(node):
 190         agg = aggregate()
 191         agg.node = node
 192         agg.loginbase = "unknown"
 193         try:
 194                 agg.loginbase = PlcSite.query.get(node.plc_siteid).plc_site_stats['login_base']
 195         except:
 196                 agg.loginbase = "exception"
 197
 198
 199         return agg
 200
 201 def prep_node_for_display(node, pcuhash=None, preppcu=True, asofdate=None):
 202         agg = aggregate()
 203         agg.node = node
 204
 205         if node.plc_pcuid and preppcu:
 206                 if pcuhash:
 207                         pcu = pcuhash[node.plc_pcuid]
 208                 else:
 209                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=node.plc_pcuid)
 210
 211                 if pcu:
 212                         agg.pcu_status = pcu.reboot_trial_status
 213                         agg.pcu_short_status = format_pcu_shortstatus(pcu)
 214                         agg.pcu = prep_pcu_for_display(pcu)
 215                 else:
 216                         agg.pcu_short_status = "none"
 217                         agg.pcu_status = "nodata"
 218                         agg.pcu = None
 219
 220         else:
 221                 agg.pcu_status = "nopcu"
 222                 agg.pcu_short_status = "none"
 223                 agg.pcu = None
 224
 225
 226         if node.kernel_version:
 227                 agg.kernel = node.kernel_version.split()[2]
 228         else:
 229                 agg.kernel = ""
 230
 231         try:
 232                 agg.loginbase = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 233         except:
 234                 agg.loginbase = "unknown"
 235
 236         if agg.loginbase:
 237                 agg.site = HistorySiteRecord.by_loginbase(agg.loginbase)
 238
 239                 if asofdate:
 240                         agg.site = agg.site.get_as_of(asofdate)
 241
 242                 if agg.site is None:
 243                         # TODO: need a cleaner fix for this...
 244                         agg.site = HistorySiteRecord.by_loginbase("pl")
 245                         if not agg.site:
 246                                 agg.site = HistorySiteRecord.by_loginbase("ple")
 247
 248         agg.history = HistoryNodeRecord.by_hostname(node.hostname)
 249         if asofdate:
 250                 agg.history = agg.history.get_as_of(asofdate)
 251
 252         agg.ports = format_ports(node.port_status)
 253
 254         try:
 255                 exists = node.plc_node_stats['last_contact']
 256         except:
 257                 # TODO: this should not assign to the fb object!
 258                 node.plc_node_stats = {'last_contact' : None}
 259
 260         return agg
 261
 262
 263 class Root(controllers.RootController, MonitorXmlrpcServer):
 264         @expose(template="monitorweb.templates.welcome")
 265         def index(self):
 266                 # log.debug("Happy TurboGears Controller Responding For Duty")
 267                 flash("Welcome To MyOps!")
 268                 return dict(now=time.ctime())
 269
 270         @expose(template="monitorweb.templates.nodelist", allow_json=True)
 271         def node3(self, filter=None):
 272                 nhquery = HistoryNodeRecord.query.all()
 273                 query = []
 274                 for nh in nhquery:
 275                         if filter:
 276                                 if nh.status == filter:
 277                                         query.append(nh)
 278                         else:
 279                                 query.append(nh)
 280
 281                 rquery=[]
 282                 for q in query:
 283                         fb = FindbadNodeRecord.get_latest_by(hostname=q.hostname)
 284                         rquery.append(fb)
 285
 286                 return dict(now=time.ctime(), query=rquery)
 287
 288         def node_query(self, filter):
 289                 nhquery = HistoryNodeRecord.query.all()
 290                 query = []
 291                 for nh in nhquery:
 292                         if filter:
 293                                 if nh.status == filter:
 294                                         query.append(nh)
 295                         else:
 296                                 query.append(nh)
 297
 298                 rquery=[]
 299                 for q in query:
 300                         fb = FindbadNodeRecord.get_latest_by(hostname=q.hostname)
 301                         agg = prep_node_for_display(fb)
 302                         rquery.append(agg)
 303                 return rquery
 304
 305         @expose("cheetah:monitorweb.templates.nodelist_plain", as_format="plain",
 306                 accept_format="text/plain", content_type="text/plain")
 307         @expose(template="monitorweb.templates.nodelist", allow_json=True)
 308         def node2(self, filter=None):
 309                 rquery=self.node_query(filter)
 310                 widget = NodeWidget(template='monitorweb.templates.node_template')
 311                 return dict(now=time.ctime(), query=rquery, nodewidget=widget)
 312
 313         @expose("cheetah:monitorweb.templates.query_plain", as_format="plain",
 314                 accept_format="text/plain", content_type="text/plain")
 315         @expose(template="monitorweb.templates.query", allow_json=True)
 316         def query(self, **data):
 317                 query = []
 318
 319                 for k in data:
 320                         print k, data[k]
 321
 322                 fbquery = None
 323
 324                 if 'object' in data and data['object'] == "nodes":
 325                         fbquery = FindbadNodeRecord.get_all_latest()
 326                 elif 'object' in data and data['object'] == "nodehistory":
 327                         hostname = data['nodehistory_hostname']
 328                         data['date_checked'] = 'date_checked'
 329                         fbrecord = FindbadNodeRecord.get_by(hostname=hostname)
 330                         fbquery = fbrecord.versions[-500:]
 331
 332                 if fbquery:
 333                         for node in fbquery:
 334                                 # NOTE: reformat some fields.
 335                                 if type(node) is not type(FindbadNodeRecord):
 336                                         agg = node.__dict__.copy()
 337                                 else:
 338                                         agg = node.to_dict()
 339                                 agg.update(agg['plc_node_stats'])
 340                                 if agg['kernel_version']:
 341                                         agg['kernel_version'] = agg['kernel_version'].split()[2]
 342                                 if 'rpmvalue' in data and 'rpms' in data:
 343                                         if agg['rpms']:
 344                                                 rpm_list = agg['rpms'].split()
 345                                                 rpm_list = filter(lambda x: data['rpmvalue'] in x, rpm_list)
 346                                                 agg['rpms'] = " ".join(rpm_list)
 347
 348                                 query.append(agg)
 349
 350                 fields=data.copy()
 351
 352                 try:
 353                         del fields['object']
 354                         del fields['rpmvalue']
 355                         del fields['nodehistory_hostname']
 356                 except: pass
 357                 return dict(now=time.ctime(), query=query, fields=fields, data=data, queryform=getNodeQueryForm())
 358
 359         @expose(template="monitorweb.templates.nodefast", allow_json=True)
 360         def node(self, filter=None):
 361                 nhquery = HistoryNodeRecord.query.all()
 362                 query = []
 363                 for nh in nhquery:
 364                         if filter:
 365                                 if nh.status == filter:
 366                                         agg = prep_nodehist(nh)
 367                                         query.append(agg)
 368                         else:
 369                                 agg = prep_nodehist(nh)
 370                                 query.append(agg)
 371
 372                 return dict(now=time.ctime(), query=query)
 373
 374         @expose(template="monitorweb.templates.nodelist")
 375         def nodeslow(self, filter='boot'):
 376                 print "NODE------------------"
 377                 print "befor-len: ", len( [ i for i in session] )
 378                 session.flush(); session.clear()
 379                 print "after-len: ", len( [ i for i in session] )
 380                 fbquery = FindbadNodeRecord.get_all_latest()
 381                 query = []
 382                 filtercount = {'down' : 0, 'boot': 0, 'debug' : 0, 'diagnose' : 0, 'disabled': 0,
 383                                                 'neverboot' : 0, 'pending' : 0, 'all' : 0, None : 0}
 384                 for node in fbquery:
 385                         # NOTE: reformat some fields.
 386                         agg = prep_node_for_display(node)
 387
 388                         if not agg.history:
 389                                 continue
 390
 391                         if agg.history.status in ['down', 'offline']:
 392                                 if node.plc_node_stats and node.plc_node_stats['last_contact'] != None:
 393                                         filtercount['down'] += 1
 394                                 else:
 395                                         filtercount['neverboot'] += 1
 396                         elif agg.history.status in ['good', 'online']:
 397                                 filtercount['boot'] += 1
 398                         elif agg.history.status in ['debug', 'monitordebug']:
 399                                 filtercount['debug'] += 1
 400                         else:
 401                                 if filtercount.has_key(agg.history.status):
 402                                         filtercount[agg.history.status] += 1
 403
 404
 405                         # NOTE: apply filter
 406                         if filter == "neverboot":
 407                                 if not node.plc_node_stats or node.plc_node_stats['last_contact'] == None:
 408                                         query.append(agg)
 409                         elif filter == "all":
 410                                 query.append(agg)
 411                         elif filter == agg.history.status:
 412                                 query.append(agg)
 413                         elif filter == 'boot':
 414                                 query.append(agg)
 415
 416
 417                 widget = NodeWidget(template='monitorweb.templates.node_template')
 418                 return dict(now=time.ctime(), query=query, fc=filtercount, nodewidget=widget)
 419
 420         def nodeaction_handler(self, tg_exceptions=None):
 421                 """Handle any kind of error."""
 422                 print "NODEACTION_HANDLER------------------"
 423
 424                 if 'pcuid' in request.params:
 425                         pcuid = request.params['pcuid']
 426                 else:
 427                         refurl = request.headers.get("Referer",link("pcu"))
 428                         print refurl
 429
 430                         # TODO: do this more intelligently...
 431                         uri_fields = urllib.splitquery(refurl)
 432                         if uri_fields[1] is not None:
 433                                 val = query_to_dict(uri_fields[1])
 434                                 if 'pcuid' in val:
 435                                         pcuid = val['pcuid']
 436                                 elif 'hostname' in val:
 437                                         pcuid = FindbadNodeRecord.get_latest_by(hostname=val['hostname']).plc_pcuid
 438                                 else:
 439                                         pcuid=None
 440                         else:
 441                                 pcuid=None
 442
 443                 cherry_trail = cherrypy._cputil.get_object_trail()
 444                 for i in cherry_trail:
 445                         print "trail: ", i
 446
 447                 print pcuid
 448                 return self.pcuview(None, pcuid, **dict(exceptions=tg_exceptions))
 449
 450         def nodeaction(self, **data):
 451                 print "NODEACTION------------------"
 452                 for item in data.keys():
 453                         print "%s %s" % ( item, data[item] )
 454
 455                 if 'hostname' in data:
 456                         hostname = data['hostname']
 457                 else:
 458                         flash("No hostname given in submitted data")
 459                         return
 460
 461                 if 'submit' in data or 'type' in data:
 462                         try:
 463                                 action = data['submit']
 464                         except:
 465                                 action = data['type']
 466                 else:
 467                         flash("No submit action given in submitted data")
 468                         return
 469
 470                 if action == "Reboot":
 471                         print "REBOOT: %s" % hostname
 472                         ret = reboot.reboot_str(str(hostname))
 473                         print ret
 474                         if ret: raise RuntimeError("Error using PCU: " + str(ret))
 475                         flash("Reboot appeared to work.  Allow at most 5 minutes.  Then run ExternalScan to check current status.")
 476
 477                 elif action == "ExternalScan":
 478                         scanapi.externalprobe(str(hostname))
 479                         flash("External Scan Successful!")
 480                 elif action == "InternalScan":
 481                         scanapi.internalprobe(str(hostname))
 482                         flash("Internal Scan Successful!")
 483                 else:
 484                         # unknown action
 485                         raise RuntimeError("Unknown action given")
 486                 return
 487
 488         @expose(template="monitorweb.templates.simpleview")
 489         def simpleview(self, **data):
 490                 return self.pre_view(**data)
 491
 492         @expose(template="monitorweb.templates.detailview")
 493         def detailview(self, **data):
 494                 return self.pre_view(**data)
 495
 496         def pre_view(self, **data):
 497                 session.flush(); session.clear()
 498
 499                 loginbase=None
 500                 loginbase_list=[]
 501                 hostname=None
 502                 pcuid=None
 503                 since=20
 504                 # if objtype is not None, then treat 'hostname' or 'loginbase' as a search pattern
 505                 objtype=None
 506
 507                 exceptions = None
 508                 sitequery=[]
 509                 nodequery=[]
 510                 pcuquery=[]
 511                 actions=[]
 512                 actions_list=[]
 513
 514                 for key in data:
 515                         print key, data[key]
 516
 517                 if 'query' in data:
 518                         obj = data['query']
 519                         fields = obj.split(":")
 520                         if len(fields) > 1:
 521                                 objtype = fields[0]
 522                                 obj = fields[1].replace("*", "%")
 523                                 print "obj: %s"% obj
 524
 525                         if len(obj.split(".")) > 1 or objtype == "node":
 526                                 hostname = obj
 527                         else:
 528                                 loginbase = obj
 529
 530                 if 'loginbase' in data:
 531                         loginbase = data['loginbase']
 532
 533                 if 'hostname' in data:
 534                         hostname = data['hostname']
 535
 536                 if 'pcuid' in data:
 537                         try: pcuid = int(data['pcuid'])
 538                         except: pcuid = None
 539
 540                 if 'since' in data:
 541                         try: since = int(since)
 542                         except: since = 20
 543
 544                 if pcuid:
 545                         print "pcuid: %s" % pcuid
 546                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=pcuid)
 547                         loginbase_list += [ PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base'] ]
 548
 549                 if hostname:
 550                         if not objtype:
 551                                 nodes = [ FindbadNodeRecord.get_latest_by(hostname=hostname) ]
 552                         else:
 553                                 nodes = FindbadNodeRecord.query.filter(FindbadNodeRecord.hostname.like(hostname))
 554
 555                         for node in nodes:
 556                                 lb = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 557                                 if lb not in loginbase_list:
 558                                         loginbase_list += [ lb ]
 559
 560                 if loginbase:
 561                         if not objtype:
 562                                 loginbase_list = [ loginbase ]
 563                         else:
 564                                 loginbase_list = HistorySiteRecord.query.filter(HistorySiteRecord.loginbase.like(loginbase))
 565                                 loginbase_list = [ l.loginbase for l in loginbase_list ]
 566
 567
 568                 if loginbase_list:
 569                         for loginbase in loginbase_list:
 570                                 actions = ActionRecord.query.filter_by(loginbase=loginbase
 571                                                                 ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since)
 572                                                                 ).order_by(ActionRecord.date_created.desc())
 573                                 actions_list += [ a for a in actions ]
 574                                 site = HistorySiteRecord.by_loginbase(loginbase)
 575                                 if site:
 576                                         sitequery.append(site)
 577                                 # NOTE: because a single pcu may be assigned to multiple hosts,
 578                                 # track unique pcus by their plc_pcuid, then turn dict into list
 579                                 pcus = {}
 580                                 for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 581                                                 # NOTE: reformat some fields.
 582                                                 agg = prep_node_for_display(node)
 583                                                 nodequery += [agg]
 584                                                 if agg.pcu:
 585                                                         pcus[agg.pcu.pcu.plc_pcuid] = agg.pcu
 586
 587                                 for pcuid_key in pcus:
 588                                         pcuquery += [pcus[pcuid_key]]
 589
 590                 actionlist_widget = ActionListWidget(template='monitorweb.templates.actionlist_template')
 591                 return dict(sitequery=sitequery, pcuquery=pcuquery, nodequery=nodequery, actions=actions_list, actionlist_widget=actionlist_widget, since=since, exceptions=exceptions)
 592
 593
 594         # TODO: add form validation
 595         @expose(template="monitorweb.templates.pcuview")
 596         @exception_handler(nodeaction_handler,"isinstance(tg_exceptions,RuntimeError)")
 597         def pcuview(self, loginbase=None, pcuid=None, hostname=None, since=20, **data):
 598                 session.flush(); session.clear()
 599                 sitequery=[]
 600                 pcuquery=[]
 601                 nodequery=[]
 602                 actions=[]
 603                 exceptions = None
 604
 605                 try: since = int(since)
 606                 except: since = 7
 607
 608                 for key in data:
 609                         print key, data[key]
 610
 611                 if 'submit' in data.keys() or 'type' in data.keys():
 612                         if hostname: data['hostname'] = hostname
 613                         self.nodeaction(**data)
 614                 if 'exceptions' in data:
 615                         exceptions = data['exceptions']
 616
 617                 if 'query' in data:
 618                         obj = data['query']
 619                         if len(obj.split(".")) > 1: hostname = obj
 620                         else: loginbase=obj
 621
 622                 if pcuid:
 623                         print "pcuid: %s" % pcuid
 624                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=pcuid)
 625                         loginbase = PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base']
 626
 627                 if hostname:
 628                         node = FindbadNodeRecord.get_latest_by(hostname=hostname)
 629                         loginbase = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 630
 631                 if loginbase:
 632                         actions = ActionRecord.query.filter_by(loginbase=loginbase
 633                                                         ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since)
 634                                                         ).order_by(ActionRecord.date_created.desc())
 635                         actions = [ a for a in actions ]
 636                         sitequery = [HistorySiteRecord.by_loginbase(loginbase)]
 637                         pcus = {}
 638                         for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 639                                         # NOTE: reformat some fields.
 640                                         agg = prep_node_for_display(node)
 641                                         nodequery += [agg]
 642                                         if agg.pcu: #.pcu.plc_pcuid:    # not None
 643                                                 #pcu = FindbadPCURecord.get_latest_by(plc_pcuid=agg.plc_pcuid)
 644                                                 #prep_pcu_for_display(pcu)
 645                                                 pcus[agg.pcu.pcu.plc_pcuid] = agg.pcu
 646
 647                         for pcuid_key in pcus:
 648                                 pcuquery += [pcus[pcuid_key]]
 649
 650                 return dict(sitequery=sitequery, pcuquery=pcuquery, nodequery=nodequery, actions=actions, since=since, exceptions=exceptions)
 651
 652         @expose(template="monitorweb.templates.pcuhistory")
 653         def pcuhistory(self, pcu_id=None):
 654                 query = []
 655                 if pcu_id:
 656                         fbnode = HistoryPCURecord.get_by(plc_pcuid=pcu_id)
 657                         l = fbnode.versions[-100:]
 658                         l.reverse()
 659                         for pcu in l:
 660                                 #prep_node_for_display(node)
 661                                 query.append(pcu)
 662
 663                 return dict(query=query, pcu_id=pcu_id)
 664
 665         @expose(template="monitorweb.templates.nodescanhistory")
 666         def nodescanhistory(self, hostname=None, length=10):
 667                 try: length = int(length)
 668                 except: length = 21
 669
 670                 fbnode = FindbadNodeRecord.get_by(hostname=hostname)
 671                 # TODO: add links for earlier history if desired.
 672                 l = fbnode.versions[-length:]
 673                 l.reverse()
 674                 query=[]
 675                 for node in l:
 676                         agg = prep_node_for_display(node, pcuhash=None, preppcu=False, asofdate=node.timestamp)
 677                         query.append(agg)
 678
 679                 if 'length' in request.params:
 680                         del request.params['length']
 681                 return dict(query=query, hostname=hostname, params=request.params)
 682
 683         @expose(template="monitorweb.templates.nodehistory")
 684         def nodehistory(self, hostname=None):
 685                 query = []
 686                 if hostname:
 687                         fbnode = HistoryNodeRecord.get_by(hostname=hostname)
 688                         l = fbnode.versions[-100:]
 689                         l.reverse()
 690                         for node in l:
 691                                 #prep_node_for_display(node)
 692                                 query.append(node)
 693
 694                 return dict(query=query, hostname=hostname)
 695
 696         @expose(template="monitorweb.templates.sitehistory")
 697         def sitehistory(self, loginbase=None):
 698                 query = []
 699                 if loginbase:
 700                         fbsite = HistorySiteRecord.get_by(loginbase=loginbase)
 701                         # TODO: add links for earlier history if desired.
 702                         l = fbsite.versions[-100:]
 703                         l.reverse()
 704                         for site in l:
 705                                 query.append(site)
 706                 return dict(query=query, loginbase=loginbase)
 707
 708
 709         @expose(template="monitorweb.templates.pculist")
 710         def pcu(self, filter='all'):
 711                 print "PCUVIEW------------------"
 712                 print "befor-len: ", len( [ i for i in session] )
 713                 session.flush(); session.clear()
 714                 print "after-len: ", len( [ i for i in session] )
 715                 fbquery = FindbadPCURecord.get_all_latest()
 716                 query = []
 717                 filtercount = {'ok' : 0, 'NetDown': 0, 'Not_Run' : 0, 'pending' : 0, 'all' : 0}
 718                 for node in fbquery:
 719
 720                         # NOTE: count filter
 721                         if node.reboot_trial_status == str(0):
 722                                 filtercount['ok'] += 1
 723                         elif node.reboot_trial_status == 'NetDown' or node.reboot_trial_status == 'Not_Run':
 724                                 filtercount[node.reboot_trial_status] += 1
 725                         else:
 726                                 filtercount['pending'] += 1
 727
 728                         pcuagg = prep_pcu_for_display(node)
 729
 730                         # NOTE: apply filter
 731                         if filter == "all":
 732                                 query.append(pcuagg)
 733                         elif filter == "ok" and node.reboot_trial_status == str(0):
 734                                 query.append(pcuagg)
 735                         elif filter == node.reboot_trial_status:
 736                                 query.append(pcuagg)
 737                         elif filter == "pending":
 738                                 # TODO: look in message logs...
 739                                 if node.reboot_trial_status != str(0) and \
 740                                         node.reboot_trial_status != 'NetDown' and \
 741                                         node.reboot_trial_status != 'Not_Run':
 742
 743                                         query.append(pcuagg)
 744
 745                 return dict(query=query, fc=filtercount)
 746
 747         @expose(template="monitorweb.templates.sitelist")
 748         def site(self, filter='all'):
 749                 print "SITE------------------"
 750                 print "befor-len: ", len( [ i for i in session] )
 751                 session.flush(); session.clear()
 752                 print "after-len: ", len( [ i for i in session] )
 753                 filtercount = {'good' : 0, 'down': 0, 'online':0, 'offline' : 0, 'new' : 0, 'pending' : 0, 'all' : 0}
 754                 fbquery = HistorySiteRecord.query.all()
 755                 query = []
 756                 for site in fbquery:
 757                         # count filter
 758                         filtercount['all'] += 1
 759                         if site.new and site.slices_used == 0 and not site.enabled:
 760                                 filtercount['new'] += 1
 761                         elif not site.enabled:
 762                                 filtercount['pending'] += 1
 763                         elif site.status in ['good', 'online']:
 764                                 filtercount['good'] += 1
 765                         elif site.status in ['down', 'offline']:
 766                                 filtercount['down'] += 1
 767
 768                         # apply filter
 769                         if filter == "all":
 770                                 query.append(site)
 771                         elif filter == 'new' and site.new and site.slices_used == 0 and not site.enabled:
 772                                 query.append(site)
 773                         elif filter == "pending" and not site.enabled:
 774                                 query.append(site)
 775                         elif filter == 'good' and site.status in ['good', 'online']:
 776                                 query.append(site)
 777                         elif filter == 'down' and site.status in ['down', 'offline']:
 778                                 query.append(site)
 779
 780                 return dict(query=query, fc=filtercount)
 781         @expose(template="monitorweb.templates.sitesummary")
 782         def sitesummary(self, loginbase="princeton"):
 783                 nodequery = []
 784                 for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 785                         agg = prep_node_for_display(node)
 786                         nodequery += [agg]
 787
 788                 return dict(nodequery=nodequery, loginbase=loginbase)
 789
 790         @expose(template="monitorweb.templates.summary")
 791         def summary(self, since=7):
 792                 sumdata = {}
 793                 sumdata['nodes'] = {}
 794                 sumdata['sites'] = {}
 795                 sumdata['pcus'] = {}
 796
 797                 def summarize(query, type):
 798                         for o in query:
 799                                 if o.status not in sumdata[type]:
 800                                         sumdata[type][o.status] = 0
 801                                 sumdata[type][o.status] += 1
 802
 803                 fbquery = HistorySiteRecord.query.all()
 804                 summarize(fbquery, 'sites')
 805                 fbquery = HistoryPCURecord.query.all()
 806                 summarize(fbquery, 'pcus')
 807                 fbquery = HistoryNodeRecord.query.all()
 808                 summarize(fbquery, 'nodes')
 809
 810                 if 'monitordebug' in sumdata['nodes']:
 811                         d = sumdata['nodes']['monitordebug']
 812                         del sumdata['nodes']['monitordebug']
 813                         sumdata['nodes']['failboot'] = d
 814
 815                 return dict(sumdata=sumdata, setorder=['good', 'offline', 'down', 'online'])
 816
 817         @expose(template="monitorweb.templates.actionsummary")
 818         def actionsummary(self, since=7):
 819                 from monitor.wrapper.emailTxt import mailtxt
 820
 821                 types = filter(lambda x: 'notice' in x, dir(mailtxt))
 822                 results = {}
 823
 824                 print mon_metadata.bind
 825                 if session.bind is None:
 826                         #TODO: figure out why this value gets cleared out...
 827                         session.bind = mon_metadata.bind
 828                 result = session.execute("select distinct(action_type) from actionrecord;")
 829
 830                 types = [r[0] for r in result]
 831
 832                 try: since = int(since)
 833                 except: since = 7
 834
 835                 for  t in types:
 836                         acts = ActionRecord.query.filter(ActionRecord.action_type==t
 837                                         ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since))
 838                         results[t] = acts.count()
 839                 return dict(results=results)
 840
 841         @expose(template="monitorweb.templates.actionlist")
 842         def actionlist(self, since=7, action_type=None, loginbase=None):
 843
 844                 try: since = int(since)
 845                 except: since = 7
 846
 847                 acts_query = ActionRecord.query.filter(
 848                                           ActionRecord.date_created >= datetime.now() - timedelta(since)
 849                                          )
 850                 if loginbase:
 851                         acts_query = acts_query.filter_by(loginbase=loginbase)
 852
 853                 if action_type:
 854                         acts_query = acts_query.filter(ActionRecord.action_type==action_type)
 855
 856                 acts = acts_query.order_by(ActionRecord.date_created.desc())
 857
 858                 query = [ a for a in acts ]
 859
 860                 return dict(actions=query, action_type=action_type, since=since)
 861
 862         @cherrypy.expose()
 863         def upload(self, log, **keywords):
 864                 hostname = None
 865                 logtype = None
 866                 logtype_list = ['bm.log', ]
 867
 868                 if 'hostname' in keywords:
 869                         hostname = keywords['hostname']
 870                 if 'type' in keywords and keywords['type'] in logtype_list:
 871                         logtype = keywords['type']
 872
 873                 if not hostname: return ""
 874                 if not logtype: return "unknown logtype: %s" % logtype
 875
 876                 short_target_filename = bootman.bootmanager_log_name(hostname)
 877                 abs_target_filename = os.path.join(config.MONITOR_BOOTMANAGER_LOG, short_target_filename)
 878                 print "write data: %s" % abs_target_filename
 879                 util.file.dumpFile(abs_target_filename, log.file.read())
 880                 bootman.bootmanager_log_action(hostname, short_target_filename, logtype)
 881                 session.flush()
 882
 883                 print "redirecting 3"
 884
 885                 return dict()