web/MonitorWeb/monitorweb/controllers.py

   1 import turbogears as tg
   2 from turbogears import controllers, expose, flash, exception_handler, redirect
   3 from turbogears import widgets
   4 from cherrypy import request, response
   5 import cherrypy
   6 # from monitorweb import model
   7 # import logging
   8 # log = logging.getLogger("monitorweb.controllers")
   9 import re
  10 import os
  11 from monitor.database.info.model import *
  12 #from monitor.database.zabbixapi.model import *
  13 from monitor_xmlrpc import MonitorXmlrpcServer
  14 from controllers_local import LocalExtensions
  15 from pcucontrol.reboot import pcu_name
  16
  17 from monitor import util
  18 from monitor import reboot
  19 from monitor import bootman
  20 from monitor import scanapi
  21 from monitor import config
  22 import time
  23
  24 from monitor.wrapper.plccache import plcdb_hn2lb as site_hn2lb
  25
  26 from monitorweb.templates.links import *
  27
  28 def session_clear_all():
  29     session.flush()
  30     try:
  31         session.expunge_all()
  32     except AttributeError: # SQLAlchemy < 0.5.1
  33         session.clear()
  34
  35 class ObjectQueryFields(widgets.WidgetsList):
  36         """The WidgetsList defines the fields of the form."""
  37         pass
  38
  39
  40
  41 class NodeQueryFields(widgets.WidgetsList):
  42         """The WidgetsList defines the fields of the form."""
  43
  44         object = widgets.RadioButtonList(label="Query Type", options=[('nodes', 'All Nodes'),
  45                                                                                                                           ('nodehistory', 'Single Node History'),
  46                                                                                                                           #('sites', 'All Sites'),
  47                                                                                                                           #('sitehistory', 'Single Site History'),
  48                                                                                                                           ], default="nodes")
  49         nodehistory_hostname = widgets.TextField(label="Hostname Node History", attrs={'size':30})
  50
  51         hostname = widgets.CheckBox(label="Hostname")
  52         firewall = widgets.CheckBox(label="Firewall?")
  53         fs_status = widgets.CheckBox(label="Filesystem Status")
  54         ssh_status = widgets.CheckBox(label="SSH Status")
  55         ssh_error = widgets.CheckBox(label="SSH Errors")
  56         dns_status = widgets.CheckBox(label="DNS Status")
  57         iptables_status = widgets.CheckBox(label="IP Tables Status")
  58         nm_status = widgets.CheckBox(label="NM Status")
  59         princeton_comon_dir = widgets.CheckBox(label="CoMon Dir")
  60         princeton_comon_running = widgets.CheckBox(label="CoMon Running")
  61         princeton_comon_procs = widgets.CheckBox(label="CoMon Processes")
  62         external_dns_status = widgets.CheckBox(label="Hostname Resolves?")
  63         kernel_version = widgets.CheckBox(label="Kernel")
  64         bootcd_version = widgets.CheckBox(label="BootCD")
  65         boot_server = widgets.CheckBox(label="Boot Server")
  66         install_date = widgets.CheckBox(label="Installation Date")
  67         observed_status = widgets.CheckBox(label="Observed Status")
  68         uptime = widgets.CheckBox(label="Uptime")
  69         traceroute = widgets.CheckBox(label="Traceroute")
  70         port_status = widgets.CheckBox(label="Port Status")
  71         plc_pcuid = widgets.CheckBox(label="PCU ID")
  72         rpms = widgets.CheckBox(label="RPM")
  73         rpmvalue = widgets.TextField(label="RPM Pattern")
  74
  75 class QueryForm(widgets.TableForm):
  76     template = """
  77     <form xmlns:py="http://purl.org/kid/ns#"
  78         id="queryform"
  79         name="${name}"
  80         action="${action}"
  81         method="${method}"
  82         class="tableform"
  83         py:attrs="form_attrs"
  84     >
  85         <div py:for="field in hidden_fields"
  86             py:replace="field.display(value_for(field), **params_for(field))"
  87         />
  88         <table border="0" cellspacing="0" cellpadding="2" py:attrs="table_attrs">
  89             <tr py:for="i, field in enumerate(fields)"
  90                 class="${i%2 and 'odd' or 'even'}"
  91             >
  92                 <th>
  93                     <label class="fieldlabel" for="${field.field_id}" py:content="field.label" />
  94                 </th>
  95                 <td>
  96                     <span py:replace="field.display(value_for(field), **params_for(field))" />
  97                     <span py:if="error_for(field)" class="fielderror" py:content="error_for(field)" />
  98                     <span py:if="field.help_text" class="fieldhelp" py:content="field.help_text" />
  99                 </td>
 100             </tr>
 101             <tr>
 102                 <td>&#160;</td>
 103                 <td py:content="submit.display(submit_text)" />
 104             </tr>
 105         </table>
 106     </form>
 107         """
 108
 109 def getNodeQueryForm():
 110         return QueryForm(fields=NodeQueryFields(), action="query")
 111
 112 # make it easier group objects without invoking the elixir auto-write feature.
 113 class aggregate: pass
 114
 115
 116 def query_to_dict(query):
 117         """ take a url query string and chop it up """
 118         val = {}
 119         query_fields = query.split('&')
 120         for f in query_fields:
 121                 (k,v) = urllib.splitvalue(f)
 122                 val[k] = v
 123
 124         return val
 125
 126 def format_ports(data, pcumodel=None):
 127         retval = []
 128         filtered_length=0
 129
 130         if pcumodel:
 131                 supported_ports=reboot.model_to_object(pcumodel).supported_ports
 132         else:
 133                 # ports of a production node
 134                 supported_ports=[22,80,806]
 135
 136         if data and len(data.keys()) > 0 :
 137                 for port in supported_ports:
 138                         try:
 139                                 state = data[str(port)]
 140                         except:
 141                                 state = "unknown"
 142
 143                         if state == "filtered":
 144                                 filtered_length += 1
 145
 146                         retval.append( (port, state) )
 147
 148         if retval == []:
 149                 retval = [( "Closed/Filtered", "state" )]
 150
 151         if filtered_length == len(supported_ports):
 152                 retval = [( "All Filtered", "state" )]
 153
 154         return retval
 155
 156 def format_pcu_shortstatus(pcu):
 157         status = "error"
 158         if pcu:
 159                 if pcu.reboot_trial_status == str(0) or pcu.reboot_trial_status == "Test: No error":
 160                         status = "Ok"
 161                 elif pcu.reboot_trial_status == "NetDown" or pcu.reboot_trial_status == "Not_Run":
 162                         status = pcu.reboot_trial_status
 163                 else:
 164                         status = "error"
 165
 166         return status
 167
 168 def prep_pcu_for_display(pcu):
 169         agg = aggregate()
 170         agg.pcu = pcu
 171
 172         try:
 173                 agg.loginbase = PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base']
 174         except:
 175                 agg.loginbase = "unknown"
 176
 177         agg.pcuhist = HistoryPCURecord.query.get(pcu.plc_pcuid)
 178
 179         agg.ports = format_ports(pcu.port_status, pcu.plc_pcu_stats['model'])
 180         agg.status = format_pcu_shortstatus(pcu)
 181         agg.pcu_name = pcu_name(pcu.plc_pcu_stats)
 182
 183         #print pcu.entry_complete
 184         agg.entry_complete_str = pcu.entry_complete
 185         #pcu.entry_complete_str += "".join([ f[0] for f in pcu.entry_complete.split() ])
 186         if pcu.dns_status == "NOHOSTNAME":
 187                 agg.dns_short_status = 'NoHost'
 188         elif pcu.dns_status == "DNS-OK":
 189                 agg.dns_short_status = 'Ok'
 190         elif pcu.dns_status == "DNS-NOENTRY":
 191                 agg.dns_short_status = 'NoEntry'
 192         elif pcu.dns_status == "NO-DNS-OR-IP":
 193                 agg.dns_short_status = 'NoHostOrIP'
 194         elif pcu.dns_status == "DNS-MISMATCH":
 195                 agg.dns_short_status = 'Mismatch'
 196         return agg
 197
 198 class ActionListWidget(widgets.Widget):
 199         pass
 200
 201 class NodeWidget(widgets.Widget):
 202         pass
 203
 204 def prep_nodehist(node):
 205         agg = aggregate()
 206         agg.node = node
 207         agg.loginbase = "unknown"
 208         try:
 209                 agg.loginbase = PlcSite.query.get(node.plc_siteid).plc_site_stats['login_base']
 210         except:
 211                 agg.loginbase = "exception"
 212
 213
 214         return agg
 215
 216 def prep_node_for_display(node, pcuhash=None, preppcu=True, asofdate=None):
 217         agg = aggregate()
 218         agg.node = node
 219
 220         if node.plc_pcuid and preppcu:
 221                 if pcuhash:
 222                         pcu = pcuhash[node.plc_pcuid]
 223                 else:
 224                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=node.plc_pcuid)
 225
 226                 if pcu:
 227                         agg.pcu_status = pcu.reboot_trial_status
 228                         agg.pcu_short_status = format_pcu_shortstatus(pcu)
 229                         agg.pcu = prep_pcu_for_display(pcu)
 230                 else:
 231                         agg.pcu_short_status = "none"
 232                         agg.pcu_status = "nodata"
 233                         agg.pcu = None
 234
 235         else:
 236                 agg.pcu_status = "nopcu"
 237                 agg.pcu_short_status = "none"
 238                 agg.pcu = None
 239
 240
 241         if node.kernel_version:
 242                 agg.kernel = node.kernel_version.split()[2]
 243         else:
 244                 agg.kernel = ""
 245
 246         try:
 247                 agg.loginbase = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 248         except:
 249                 agg.loginbase = "unknown"
 250
 251         if agg.loginbase:
 252                 agg.site = HistorySiteRecord.by_loginbase(agg.loginbase)
 253
 254                 if asofdate:
 255                         agg.site = agg.site.get_as_of(asofdate)
 256
 257                 if agg.site is None:
 258                         # TODO: need a cleaner fix for this...
 259                         agg.site = HistorySiteRecord.by_loginbase("pl")
 260                         if not agg.site:
 261                                 agg.site = HistorySiteRecord.by_loginbase("ple")
 262
 263         agg.history = HistoryNodeRecord.by_hostname(node.hostname)
 264         if asofdate:
 265                 agg.history = agg.history.get_as_of(asofdate)
 266
 267         agg.ports = format_ports(node.port_status)
 268
 269         try:
 270                 exists = node.plc_node_stats['last_contact']
 271         except:
 272                 # TODO: this should not assign to the fb object!
 273                 node.plc_node_stats = {'last_contact' : None}
 274
 275         return agg
 276
 277
 278 class Root(controllers.RootController, MonitorXmlrpcServer, LocalExtensions):
 279         @expose(template="monitorweb.templates.welcome")
 280         def index(self):
 281                 # log.debug("Happy TurboGears Controller Responding For Duty")
 282                 flash("Welcome To MyOps!")
 283                 return dict(now=time.ctime())
 284
 285         @expose(template="monitorweb.templates.nodelist", allow_json=True)
 286         def node3(self, filter=None):
 287                 nhquery = HistoryNodeRecord.query.all()
 288                 query = []
 289                 for nh in nhquery:
 290                         if filter:
 291                                 if nh.status == filter:
 292                                         query.append(nh)
 293                         else:
 294                                 query.append(nh)
 295
 296                 rquery=[]
 297                 for q in query:
 298                         fb = FindbadNodeRecord.get_latest_by(hostname=q.hostname)
 299                         rquery.append(fb)
 300
 301                 return dict(now=time.ctime(), query=rquery)
 302
 303         def node_query(self, filter):
 304                 nhquery = HistoryNodeRecord.query.all()
 305                 query = []
 306                 for nh in nhquery:
 307                         if filter:
 308                                 if nh.status == filter:
 309                                         query.append(nh)
 310                         else:
 311                                 query.append(nh)
 312
 313                 rquery=[]
 314                 for q in query:
 315                         fb = FindbadNodeRecord.get_latest_by(hostname=q.hostname)
 316                         agg = prep_node_for_display(fb)
 317                         rquery.append(agg)
 318                 return rquery
 319
 320         @expose("cheetah:monitorweb.templates.nodelist_plain", as_format="plain",
 321                 accept_format="text/plain", content_type="text/plain")
 322         @expose(template="monitorweb.templates.nodelist", allow_json=True)
 323         def node2(self, filter=None):
 324                 rquery=self.node_query(filter)
 325                 widget = NodeWidget(template='monitorweb.templates.node_template')
 326                 return dict(now=time.ctime(), query=rquery, nodewidget=widget)
 327
 328         @expose("cheetah:monitorweb.templates.query_plain", as_format="plain",
 329                 accept_format="text/plain", content_type="text/plain")
 330         @expose(template="monitorweb.templates.query", allow_json=True)
 331         def query(self, **data):
 332                 query = []
 333
 334                 for k in data:
 335                         print k, data[k]
 336
 337                 fbquery = None
 338
 339                 if 'object' in data and data['object'] == "nodes":
 340                         fbquery = FindbadNodeRecord.get_all_latest()
 341                 elif 'object' in data and data['object'] == "nodehistory":
 342                         hostname = data['nodehistory_hostname']
 343                         data['date_checked'] = 'date_checked'
 344                         fbrecord = FindbadNodeRecord.get_by(hostname=hostname)
 345                         fbquery = fbrecord.versions[-500:]
 346
 347                 if fbquery:
 348                         for node in fbquery:
 349                                 # NOTE: reformat some fields.
 350                                 if type(node) is not type(FindbadNodeRecord):
 351                                         agg = node.__dict__.copy()
 352                                 else:
 353                                         agg = node.to_dict()
 354                                 if agg['plc_node_stats']:
 355                                         agg.update(agg['plc_node_stats'])
 356                                 if agg['install_date']:
 357                                         agg['install_date'] = time.mktime(time.strptime(agg['install_date'], "%a %b %d %H:%M:%S %Y"))
 358                                 if agg['kernel_version']:
 359                                         agg['kernel_version'] = agg['kernel_version'].split()[2]
 360                                 if 'traceroute' in data and agg['traceroute']:
 361                                         agg['traceroute'] = "<pre>" + agg['traceroute'] + "</pre>"
 362                                 if 'rpmvalue' in data and 'rpms' in data:
 363                                         if agg['rpms']:
 364                                                 rpm_list = agg['rpms'].split()
 365                                                 rpm_list = filter(lambda x: re.match(data['rpmvalue'], x, re.I),
 366                                                                   rpm_list)
 367                                                 agg['rpms'] = " ".join(rpm_list)
 368
 369                                 query.append(agg)
 370
 371                 fields=data.copy()
 372
 373                 try:
 374                         del fields['object']
 375                         del fields['rpmvalue']
 376                         del fields['nodehistory_hostname']
 377                 except: pass
 378                 return dict(now=time.ctime(), query=query, fields=fields, data=data, queryform=getNodeQueryForm())
 379
 380         @expose(template="monitorweb.templates.nodefast", allow_json=True)
 381         def node(self, filter=None):
 382                 nhquery = HistoryNodeRecord.query.all()
 383                 query = []
 384                 for nh in nhquery:
 385                         if filter:
 386                                 if nh.status == filter:
 387                                         agg = prep_nodehist(nh)
 388                                         query.append(agg)
 389                         else:
 390                                 agg = prep_nodehist(nh)
 391                                 query.append(agg)
 392
 393                 return dict(now=time.ctime(), query=query)
 394
 395         @expose(template="monitorweb.templates.nodelist")
 396         def nodeslow(self, filter='boot'):
 397                 print "NODE------------------"
 398                 print "befor-len: ", len( [ i for i in session] )
 399         session_clear_all()
 400                 print "after-len: ", len( [ i for i in session] )
 401                 fbquery = FindbadNodeRecord.get_all_latest()
 402                 query = []
 403                 filtercount = {'down' : 0, 'boot': 0, 'debug' : 0, 'diagnose' : 0, 'disabled': 0,
 404                                                 'neverboot' : 0, 'pending' : 0, 'all' : 0, None : 0}
 405                 for node in fbquery:
 406                         # NOTE: reformat some fields.
 407                         agg = prep_node_for_display(node)
 408
 409                         if not agg.history:
 410                                 continue
 411
 412                         if agg.history.status in ['down', 'offline']:
 413                                 if node.plc_node_stats and node.plc_node_stats['last_contact'] != None:
 414                                         filtercount['down'] += 1
 415                                 else:
 416                                         filtercount['neverboot'] += 1
 417                         elif agg.history.status in ['good', 'online']:
 418                                 filtercount['boot'] += 1
 419                         elif agg.history.status in ['debug', 'monitordebug']:
 420                                 filtercount['debug'] += 1
 421                         else:
 422                                 if filtercount.has_key(agg.history.status):
 423                                         filtercount[agg.history.status] += 1
 424
 425
 426                         # NOTE: apply filter
 427                         if filter == "neverboot":
 428                                 if not node.plc_node_stats or node.plc_node_stats['last_contact'] == None:
 429                                         query.append(agg)
 430                         elif filter == "all":
 431                                 query.append(agg)
 432                         elif filter == agg.history.status:
 433                                 query.append(agg)
 434                         elif filter == 'boot':
 435                                 query.append(agg)
 436
 437
 438                 widget = NodeWidget(template='monitorweb.templates.node_template')
 439                 return dict(now=time.ctime(), query=query, fc=filtercount, nodewidget=widget)
 440
 441         def nodeaction_handler(self, tg_exceptions=None):
 442                 """Handle any kind of error."""
 443                 print "NODEACTION_HANDLER------------------"
 444
 445                 if 'pcuid' in request.params:
 446                         pcuid = request.params['pcuid']
 447                 else:
 448                         refurl = request.headers.get("Referer",link("pcu"))
 449                         print refurl
 450
 451                         # TODO: do this more intelligently...
 452                         uri_fields = urllib.splitquery(refurl)
 453                         if uri_fields[1] is not None:
 454                                 val = query_to_dict(uri_fields[1])
 455                                 if 'pcuid' in val:
 456                                         pcuid = val['pcuid']
 457                                 elif 'hostname' in val:
 458                                         pcuid = FindbadNodeRecord.get_latest_by(hostname=val['hostname']).plc_pcuid
 459                                 else:
 460                                         pcuid=None
 461                         else:
 462                                 pcuid=None
 463
 464                 cherry_trail = cherrypy._cputil.get_object_trail()
 465                 for i in cherry_trail:
 466                         print "trail: ", i
 467
 468                 print pcuid
 469                 return self.pcuview(None, pcuid, **dict(exceptions=tg_exceptions))
 470
 471         def nodeaction(self, **data):
 472                 print "NODEACTION------------------"
 473                 for item in data.keys():
 474                         print "%s %s" % ( item, data[item] )
 475
 476                 if 'hostname' in data:
 477                         hostname = data['hostname']
 478                 else:
 479                         flash("No hostname given in submitted data")
 480                         return
 481
 482                 if 'submit' in data or 'type' in data:
 483                         try:
 484                                 action = data['submit']
 485                         except:
 486                                 action = data['type']
 487                 else:
 488                         flash("No submit action given in submitted data")
 489                         return
 490
 491                 if action == "Reboot":
 492                         print "REBOOT: %s" % hostname
 493                         ret = reboot.reboot_str(str(hostname))
 494                         print ret
 495                         if ret: raise RuntimeError("Error using PCU: " + str(ret))
 496                         flash("Reboot appeared to work.  Allow at most 5 minutes.  Then run ExternalScan to check current status.")
 497
 498                 elif action == "ExternalScan":
 499                         scanapi.externalprobe(str(hostname))
 500                         flash("External Scan Successful!")
 501                 elif action == "InternalScan":
 502                         scanapi.internalprobe(str(hostname))
 503                         flash("Internal Scan Successful!")
 504                 else:
 505                         # unknown action
 506                         raise RuntimeError("Unknown action given")
 507                 return
 508
 509         @expose(template="monitorweb.templates.simpleview")
 510         def simpleview(self, **data):
 511                 return self.pre_view(**data)
 512
 513         @expose(template="monitorweb.templates.simpleview")
 514         def pcuview(self, **data):
 515                 return self.pre_view(**data)
 516
 517         @expose(template="monitorweb.templates.detailview")
 518         def detailview(self, **data):
 519                 return self.pre_view(**data)
 520
 521
 522         def pre_view(self, **data):
 523         session_clear_all()
 524
 525                 loginbase=None
 526                 loginbase_list=[]
 527                 hostname=None
 528                 pcuid=None
 529                 since=20
 530                 # if objtype is not None, then treat 'hostname' or 'loginbase' as a search pattern
 531                 objtype=None
 532
 533                 exceptions = None
 534                 sitequery=[]
 535                 nodequery=[]
 536                 pcuquery=[]
 537                 actions=[]
 538                 actions_list=[]
 539
 540                 for key in data:
 541                         print key, data[key]
 542
 543                 if 'query' in data:
 544                         obj = data['query']
 545                         fields = obj.split(":")
 546                         if len(fields) > 1:
 547                                 objtype = fields[0]
 548                                 obj = fields[1].replace("*", "%")
 549                                 print "obj: %s"% obj
 550
 551                         if len(obj.split(".")) > 1 or objtype == "node":
 552                                 hostname = obj
 553                         else:
 554                                 loginbase = obj
 555
 556                 if 'loginbase' in data:
 557                         loginbase = data['loginbase']
 558
 559                 if 'hostname' in data:
 560                         hostname = data['hostname']
 561
 562                 if 'pcuid' in data:
 563                         try: pcuid = int(data['pcuid'])
 564                         except: pcuid = None
 565
 566                 if 'since' in data:
 567                         try: since = int(since)
 568                         except: since = 20
 569
 570                 if pcuid:
 571                         print "pcuid: %s" % pcuid
 572                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=pcuid)
 573                         loginbase_list += [ PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base'] ]
 574
 575                 if hostname:
 576                         if not objtype:
 577                                 nodes = [ FindbadNodeRecord.get_latest_by(hostname=hostname) ]
 578                         else:
 579                                 nodes = FindbadNodeRecord.query.filter(FindbadNodeRecord.hostname.like(hostname))
 580
 581                         for node in nodes:
 582                                 lb = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 583                                 if lb not in loginbase_list:
 584                                         loginbase_list += [ lb ]
 585
 586                 if loginbase:
 587                         if not objtype:
 588                                 loginbase_list = [ loginbase ]
 589                         else:
 590                                 loginbase_list = HistorySiteRecord.query.filter(HistorySiteRecord.loginbase.like(loginbase))
 591                                 loginbase_list = [ l.loginbase for l in loginbase_list ]
 592
 593
 594                 if loginbase_list:
 595                         for loginbase in loginbase_list:
 596                                 actions = ActionRecord.query.filter_by(loginbase=loginbase
 597                                                                 ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since)
 598                                                                 ).order_by(ActionRecord.date_created.desc())
 599                                 actions_list += [ a for a in actions ]
 600                                 site = HistorySiteRecord.by_loginbase(loginbase)
 601                                 if site:
 602                                         sitequery.append(site)
 603                                 # NOTE: because a single pcu may be assigned to multiple hosts,
 604                                 # track unique pcus by their plc_pcuid, then turn dict into list
 605                                 pcus = {}
 606                                 for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 607                                                 # NOTE: reformat some fields.
 608                                                 agg = prep_node_for_display(node)
 609                                                 nodequery += [agg]
 610                                                 if agg.pcu:
 611                                                         pcus[agg.pcu.pcu.plc_pcuid] = agg.pcu
 612
 613                                 for pcuid_key in pcus:
 614                                         pcuquery += [pcus[pcuid_key]]
 615
 616                 #for a in nodequery:
 617                 #       print type(a.node)
 618                 #       print type(a.node.hostname)
 619                 nodequery.sort(lambda a,b: cmp(a.node.hostname,b.node.hostname))
 620                 pcuquery.sort(lambda a,b: cmp(a.pcu_name,b.pcu_name))
 621
 622                 actionlist_widget = ActionListWidget(template='monitorweb.templates.actionlist_template')
 623                 return dict(sitequery=sitequery, pcuquery=pcuquery, nodequery=nodequery, actions=actions_list, actionlist_widget=actionlist_widget, since=since, exceptions=exceptions)
 624
 625
 626         # TODO: add form validation
 627         @expose(template="monitorweb.templates.pcuview")
 628         @exception_handler(nodeaction_handler,"isinstance(tg_exceptions,RuntimeError)")
 629         def pcuviewold(self, loginbase=None, pcuid=None, hostname=None, since=20, **data):
 630         session_clear_all()
 631                 sitequery=[]
 632                 pcuquery=[]
 633                 nodequery=[]
 634                 actions=[]
 635                 exceptions = None
 636
 637                 try: since = int(since)
 638                 except: since = 7
 639
 640                 for key in data:
 641                         print key, data[key]
 642
 643                 if 'submit' in data.keys() or 'type' in data.keys():
 644                         if hostname: data['hostname'] = hostname
 645                         self.nodeaction(**data)
 646                 if 'exceptions' in data:
 647                         exceptions = data['exceptions']
 648
 649                 if 'query' in data:
 650                         obj = data['query']
 651                         if len(obj.split(".")) > 1: hostname = obj
 652                         else: loginbase=obj
 653
 654                 if pcuid:
 655                         print "pcuid: %s" % pcuid
 656                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=pcuid)
 657                         loginbase = PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base']
 658
 659                 if hostname:
 660                         node = FindbadNodeRecord.get_latest_by(hostname=hostname)
 661                         loginbase = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 662
 663                 if loginbase:
 664                         actions = ActionRecord.query.filter_by(loginbase=loginbase
 665                                                         ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since)
 666                                                         ).order_by(ActionRecord.date_created.desc())
 667                         actions = [ a for a in actions ]
 668                         sitequery = [HistorySiteRecord.by_loginbase(loginbase)]
 669                         pcus = {}
 670                         for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 671                                         # NOTE: reformat some fields.
 672                                         agg = prep_node_for_display(node)
 673                                         nodequery += [agg]
 674                                         if agg.pcu: #.pcu.plc_pcuid:    # not None
 675                                                 #pcu = FindbadPCURecord.get_latest_by(plc_pcuid=agg.plc_pcuid)
 676                                                 #prep_pcu_for_display(pcu)
 677                                                 pcus[agg.pcu.pcu.plc_pcuid] = agg.pcu
 678
 679                         for pcuid_key in pcus:
 680                                 pcuquery += [pcus[pcuid_key]]
 681
 682                 return dict(sitequery=sitequery, pcuquery=pcuquery, nodequery=nodequery, actions=actions, since=since, exceptions=exceptions)
 683
 684         @expose(template="monitorweb.templates.pcuhistory")
 685         def pcuhistory(self, pcu_id=None):
 686                 query = []
 687                 if pcu_id:
 688                         fbnode = HistoryPCURecord.get_by(plc_pcuid=pcu_id)
 689                         l = fbnode.versions[-1000:]
 690                         l.reverse()
 691                         for pcu in l:
 692                                 #prep_node_for_display(node)
 693                                 query.append(pcu)
 694
 695                 return dict(query=query, pcu_id=pcu_id)
 696
 697         @expose(template="monitorweb.templates.nodescanhistory")
 698         def nodescanhistory(self, hostname=None, length=10):
 699                 try: length = int(length)
 700                 except: length = 21
 701
 702                 fbnode = FindbadNodeRecord.get_by(hostname=hostname)
 703                 # TODO: add links for earlier history if desired.
 704                 l = fbnode.versions[-length:]
 705                 l.reverse()
 706                 query=[]
 707                 for node in l:
 708                         agg = prep_node_for_display(node, pcuhash=None, preppcu=False, asofdate=node.timestamp)
 709                         query.append(agg)
 710
 711                 if 'length' in request.params:
 712                         del request.params['length']
 713                 return dict(query=query, hostname=hostname, params=request.params)
 714
 715         @expose(template="monitorweb.templates.nodehistory")
 716         def nodehistory(self, hostname=None):
 717                 query = []
 718                 if hostname:
 719                         fbnode = HistoryNodeRecord.get_by(hostname=hostname)
 720                         l = fbnode.versions[-100:]
 721                         l.reverse()
 722                         for node in l:
 723                                 #prep_node_for_display(node)
 724                                 query.append(node)
 725
 726                 return dict(query=query, hostname=hostname)
 727
 728         @expose(template="monitorweb.templates.sitehistory")
 729         def sitehistory(self, loginbase=None):
 730                 query = []
 731                 if loginbase:
 732                         fbsite = HistorySiteRecord.get_by(loginbase=loginbase)
 733                         # TODO: add links for earlier history if desired.
 734                         l = fbsite.versions[-1000:]
 735                         l.reverse()
 736                         for site in l:
 737                                 query.append(site)
 738                 return dict(query=query, loginbase=loginbase)
 739
 740
 741         @expose("cheetah:monitorweb.templates.pculist_plain", as_format="plain",
 742                 accept_format="text/plain", content_type="text/plain")
 743         @expose(template="monitorweb.templates.pculist")
 744         def pcu(self, filter='all'):
 745                 print "PCUVIEW------------------"
 746                 print "befor-len: ", len( [ i for i in session] )
 747         session_clear_all()
 748                 print "after-len: ", len( [ i for i in session] )
 749                 fbquery = FindbadPCURecord.get_all_latest()
 750                 query = []
 751                 filtercount = {'ok' : 0, 'NetDown': 0, 'Not_Run' : 0, 'pending' : 0, 'all' : 0}
 752                 for node in fbquery:
 753
 754                         # NOTE: count filter
 755                         if node.reboot_trial_status == str(0):
 756                                 filtercount['ok'] += 1
 757                         elif node.reboot_trial_status == 'NetDown' or node.reboot_trial_status == 'Not_Run':
 758                                 filtercount[node.reboot_trial_status] += 1
 759                         else:
 760                                 filtercount['pending'] += 1
 761
 762                         pcuagg = prep_pcu_for_display(node)
 763
 764                         # NOTE: apply filter
 765                         if filter == "all":
 766                                 query.append(pcuagg)
 767                         elif filter == "ok" and node.reboot_trial_status == str(0):
 768                                 query.append(pcuagg)
 769                         elif filter == node.reboot_trial_status:
 770                                 query.append(pcuagg)
 771                         elif filter == "pending":
 772                                 # TODO: look in message logs...
 773                                 if node.reboot_trial_status != str(0) and \
 774                                         node.reboot_trial_status != 'NetDown' and \
 775                                         node.reboot_trial_status != 'Not_Run':
 776
 777                                         query.append(pcuagg)
 778
 779                 return dict(query=query, fc=filtercount)
 780
 781         @expose(template="monitorweb.templates.sitelist")
 782         def site(self, filter='all'):
 783                 print "SITE------------------"
 784                 print "befor-len: ", len( [ i for i in session] )
 785         session_clear_all()
 786                 print "after-len: ", len( [ i for i in session] )
 787                 filtercount = {'good' : 0, 'down': 0, 'online':0, 'offline' : 0, 'new' : 0, 'pending' : 0, 'all' : 0}
 788                 fbquery = HistorySiteRecord.query.all()
 789                 query = []
 790                 for site in fbquery:
 791                         # count filter
 792                         filtercount['all'] += 1
 793                         if site.new and site.slices_used == 0 and not site.enabled:
 794                                 filtercount['new'] += 1
 795                         elif not site.enabled:
 796                                 filtercount['pending'] += 1
 797                         elif site.status in ['good', 'online']:
 798                                 filtercount['good'] += 1
 799                         elif site.status in ['down', 'offline']:
 800                                 filtercount['down'] += 1
 801
 802                         # apply filter
 803                         if filter == "all":
 804                                 query.append(site)
 805                         elif filter == 'new' and site.new and site.slices_used == 0 and not site.enabled:
 806                                 query.append(site)
 807                         elif filter == "pending" and not site.enabled:
 808                                 query.append(site)
 809                         elif filter == 'good' and site.status in ['good', 'online']:
 810                                 query.append(site)
 811                         elif filter == 'down' and site.status in ['down', 'offline']:
 812                                 query.append(site)
 813
 814                 return dict(query=query, fc=filtercount)
 815         @expose(template="monitorweb.templates.sitesummary")
 816         def sitesummary(self, loginbase="princeton"):
 817                 nodequery = []
 818                 for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 819                         agg = prep_node_for_display(node)
 820                         nodequery += [agg]
 821
 822                 return dict(nodequery=nodequery, loginbase=loginbase)
 823
 824         @expose(template="monitorweb.templates.summary")
 825         def summary(self, since=7):
 826                 sumdata = {}
 827                 sumdata['nodes'] = {}
 828                 sumdata['sites'] = {}
 829                 sumdata['pcus'] = {}
 830
 831                 def summarize(query, type):
 832                         for o in query:
 833                                 if o.status not in sumdata[type]:
 834                                         sumdata[type][o.status] = 0
 835                                 sumdata[type][o.status] += 1
 836
 837                 fbquery = HistorySiteRecord.query.all()
 838                 summarize(fbquery, 'sites')
 839                 fbquery = HistoryPCURecord.query.all()
 840                 summarize(fbquery, 'pcus')
 841                 fbquery = HistoryNodeRecord.query.all()
 842                 summarize(fbquery, 'nodes')
 843
 844                 if 'monitordebug' in sumdata['nodes']:
 845                         d = sumdata['nodes']['monitordebug']
 846                         del sumdata['nodes']['monitordebug']
 847                         sumdata['nodes']['failboot'] = d
 848
 849                 return dict(sumdata=sumdata, setorder=['good', 'offline', 'down', 'online'])
 850
 851         @expose(template="monitorweb.templates.actionsummary")
 852         def actionsummary(self, since=7):
 853                 from monitor.wrapper.emailTxt import mailtxt
 854
 855                 types = filter(lambda x: 'notice' in x, dir(mailtxt))
 856                 results = {}
 857
 858                 print mon_metadata.bind
 859                 if session.bind is None:
 860                         #TODO: figure out why this value gets cleared out...
 861                         session.bind = mon_metadata.bind
 862                 result = session.execute("select distinct(action_type) from actionrecord;")
 863
 864                 types = [r[0] for r in result]
 865
 866                 try: since = int(since)
 867                 except: since = 7
 868
 869                 for  t in types:
 870                         acts = ActionRecord.query.filter(ActionRecord.action_type==t
 871                                         ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since))
 872                         results[t] = acts.count()
 873                 return dict(results=results)
 874
 875         @expose(template="monitorweb.templates.actionlist")
 876         def actionlist(self, since=7, action_type=None, loginbase=None):
 877
 878                 try: since = int(since)
 879                 except: since = 7
 880
 881                 acts_query = ActionRecord.query.filter(
 882                                           ActionRecord.date_created >= datetime.now() - timedelta(since)
 883                                          )
 884                 if loginbase:
 885                         acts_query = acts_query.filter_by(loginbase=loginbase)
 886
 887                 if action_type:
 888                         acts_query = acts_query.filter(ActionRecord.action_type==action_type)
 889
 890                 acts = acts_query.order_by(ActionRecord.date_created.desc())
 891
 892                 query = [ a for a in acts ]
 893
 894                 return dict(actions=query, action_type=action_type, since=since)
 895
 896         @cherrypy.expose()
 897         def upload(self, log, **keywords):
 898                 hostname = None
 899                 logtype = None
 900                 logtype_list = ['bm.log', ]
 901
 902                 if 'hostname' in keywords:
 903                         hostname = keywords['hostname']
 904                 if 'type' in keywords and keywords['type'] in logtype_list:
 905                         logtype = keywords['type']
 906
 907                 if not hostname: return ""
 908                 if not logtype: return "unknown logtype: %s" % logtype
 909
 910                 short_target_filename = bootman.bootmanager_log_name(hostname)
 911                 abs_target_filename = os.path.join(config.MONITOR_BOOTMANAGER_LOG, short_target_filename)
 912                 print "write data: %s" % abs_target_filename
 913                 util.file.dumpFile(abs_target_filename, log.file.read())
 914                 bootman.bootmanager_log_action(hostname, short_target_filename, logtype)
 915                 session.flush()
 916
 917                 print "redirecting 3"
 918
 919                 return dict()
 920