web/MonitorWeb/monitorweb/controllers.py

   1 import turbogears as tg
   2 from turbogears import controllers, expose, flash, exception_handler, redirect
   3 from turbogears import widgets
   4 from cherrypy import request, response
   5 import cherrypy
   6 # from monitorweb import model
   7 # import logging
   8 # log = logging.getLogger("monitorweb.controllers")
   9 import re
  10 import os
  11 from monitor.database.info.model import *
  12 #from monitor.database.zabbixapi.model import *
  13 from monitor_xmlrpc import MonitorXmlrpcServer
  14
  15 from monitor import util
  16 from monitor import reboot
  17 from monitor import bootman
  18 from monitor import scanapi
  19 from monitor import config
  20 import time
  21
  22 from monitor.wrapper.plccache import plcdb_hn2lb as site_hn2lb
  23
  24 from monitorweb.templates.links import *
  25
  26 class ObjectQueryFields(widgets.WidgetsList):
  27         """The WidgetsList defines the fields of the form."""
  28         pass
  29
  30
  31
  32 class NodeQueryFields(widgets.WidgetsList):
  33         """The WidgetsList defines the fields of the form."""
  34
  35         object = widgets.RadioButtonList(label="Query Type", options=[('nodes', 'All Nodes'),
  36                                                                                                                           ('nodehistory', 'Single Node History'),
  37                                                                                                                           #('sites', 'All Sites'),
  38                                                                                                                           #('sitehistory', 'Single Site History'),
  39                                                                                                                           ], default="nodes")
  40         nodehistory_hostname = widgets.TextField(label="Hostname Node History", attrs={'size':30})
  41
  42         hostname = widgets.CheckBox(label="Hostname")
  43         firewall = widgets.CheckBox(label="Firewall?")
  44         fs_status = widgets.CheckBox(label="Filesystem Status")
  45         ssh_status = widgets.CheckBox(label="SSH Status")
  46         ssh_error = widgets.CheckBox(label="SSH Errors")
  47         dns_status = widgets.CheckBox(label="DNS Status")
  48         iptables_status = widgets.CheckBox(label="IP Tables Status")
  49         nm_status = widgets.CheckBox(label="NM Status")
  50         princeton_comon_dir = widgets.CheckBox(label="CoMon Dir")
  51         princeton_comon_running = widgets.CheckBox(label="CoMon Running")
  52         princeton_comon_procs = widgets.CheckBox(label="CoMon Processes")
  53         external_dns_status = widgets.CheckBox(label="Hostname Resolves?")
  54         kernel_version = widgets.CheckBox(label="Kernel")
  55         bootcd_version = widgets.CheckBox(label="BootCD")
  56         boot_server = widgets.CheckBox(label="Boot Server")
  57         install_date = widgets.CheckBox(label="Installation Date")
  58         observed_status = widgets.CheckBox(label="Observed Status")
  59         uptime = widgets.CheckBox(label="Uptime")
  60         traceroute = widgets.CheckBox(label="Traceroute")
  61         port_status = widgets.CheckBox(label="Port Status")
  62         rpms = widgets.CheckBox(label="RPM")
  63         rpmvalue = widgets.TextField(label="RPM Pattern")
  64
  65 class QueryForm(widgets.TableForm):
  66     template = """
  67     <form xmlns:py="http://purl.org/kid/ns#"
  68         id="queryform"
  69         name="${name}"
  70         action="${action}"
  71         method="${method}"
  72         class="tableform"
  73         py:attrs="form_attrs"
  74     >
  75         <div py:for="field in hidden_fields"
  76             py:replace="field.display(value_for(field), **params_for(field))"
  77         />
  78         <table border="0" cellspacing="0" cellpadding="2" py:attrs="table_attrs">
  79             <tr py:for="i, field in enumerate(fields)"
  80                 class="${i%2 and 'odd' or 'even'}"
  81             >
  82                 <th>
  83                     <label class="fieldlabel" for="${field.field_id}" py:content="field.label" />
  84                 </th>
  85                 <td>
  86                     <span py:replace="field.display(value_for(field), **params_for(field))" />
  87                     <span py:if="error_for(field)" class="fielderror" py:content="error_for(field)" />
  88                     <span py:if="field.help_text" class="fieldhelp" py:content="field.help_text" />
  89                 </td>
  90             </tr>
  91             <tr>
  92                 <td>&#160;</td>
  93                 <td py:content="submit.display(submit_text)" />
  94             </tr>
  95         </table>
  96     </form>
  97         """
  98
  99 def getNodeQueryForm():
 100         return QueryForm(fields=NodeQueryFields(), action="query")
 101
 102 # make it easier group objects without invoking the elixir auto-write feature.
 103 class aggregate: pass
 104
 105
 106 def query_to_dict(query):
 107         """ take a url query string and chop it up """
 108         val = {}
 109         query_fields = query.split('&')
 110         for f in query_fields:
 111                 (k,v) = urllib.splitvalue(f)
 112                 val[k] = v
 113
 114         return val
 115
 116 def format_ports(data, pcumodel=None):
 117         retval = []
 118         filtered_length=0
 119
 120         if pcumodel:
 121                 supported_ports=reboot.model_to_object(pcumodel).supported_ports
 122         else:
 123                 # ports of a production node
 124                 supported_ports=[22,80,806]
 125
 126         if data and len(data.keys()) > 0 :
 127                 for port in supported_ports:
 128                         try:
 129                                 state = data[str(port)]
 130                         except:
 131                                 state = "unknown"
 132
 133                         if state == "filtered":
 134                                 filtered_length += 1
 135
 136                         retval.append( (port, state) )
 137
 138         if retval == []:
 139                 retval = [( "Closed/Filtered", "state" )]
 140
 141         if filtered_length == len(supported_ports):
 142                 retval = [( "All Filtered", "state" )]
 143
 144         return retval
 145
 146 def format_pcu_shortstatus(pcu):
 147         status = "error"
 148         if pcu:
 149                 if pcu.reboot_trial_status == str(0):
 150                         status = "Ok"
 151                 elif pcu.reboot_trial_status == "NetDown" or pcu.reboot_trial_status == "Not_Run":
 152                         status = pcu.reboot_trial_status
 153                 else:
 154                         status = "error"
 155
 156         return status
 157
 158 def prep_pcu_for_display(pcu):
 159         agg = aggregate()
 160         agg.pcu = pcu
 161
 162         try:
 163                 agg.loginbase = PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base']
 164         except:
 165                 agg.loginbase = "unknown"
 166
 167         agg.pcuhist = HistoryPCURecord.query.get(pcu.plc_pcuid)
 168
 169         agg.ports = format_ports(pcu.port_status, pcu.plc_pcu_stats['model'])
 170         agg.status = format_pcu_shortstatus(pcu)
 171
 172         #print pcu.entry_complete
 173         agg.entry_complete_str = pcu.entry_complete
 174         #pcu.entry_complete_str += "".join([ f[0] for f in pcu.entry_complete.split() ])
 175         if pcu.dns_status == "NOHOSTNAME":
 176                 agg.dns_short_status = 'NoHost'
 177         elif pcu.dns_status == "DNS-OK":
 178                 agg.dns_short_status = 'Ok'
 179         elif pcu.dns_status == "DNS-NOENTRY":
 180                 agg.dns_short_status = 'NoEntry'
 181         elif pcu.dns_status == "NO-DNS-OR-IP":
 182                 agg.dns_short_status = 'NoHostOrIP'
 183         elif pcu.dns_status == "DNS-MISMATCH":
 184                 agg.dns_short_status = 'Mismatch'
 185         return agg
 186
 187 class ActionListWidget(widgets.Widget):
 188         pass
 189
 190 class NodeWidget(widgets.Widget):
 191         pass
 192
 193 def prep_nodehist(node):
 194         agg = aggregate()
 195         agg.node = node
 196         agg.loginbase = "unknown"
 197         try:
 198                 agg.loginbase = PlcSite.query.get(node.plc_siteid).plc_site_stats['login_base']
 199         except:
 200                 agg.loginbase = "exception"
 201
 202
 203         return agg
 204
 205 def prep_node_for_display(node, pcuhash=None, preppcu=True, asofdate=None):
 206         agg = aggregate()
 207         agg.node = node
 208
 209         if node.plc_pcuid and preppcu:
 210                 if pcuhash:
 211                         pcu = pcuhash[node.plc_pcuid]
 212                 else:
 213                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=node.plc_pcuid)
 214
 215                 if pcu:
 216                         agg.pcu_status = pcu.reboot_trial_status
 217                         agg.pcu_short_status = format_pcu_shortstatus(pcu)
 218                         agg.pcu = prep_pcu_for_display(pcu)
 219                 else:
 220                         agg.pcu_short_status = "none"
 221                         agg.pcu_status = "nodata"
 222                         agg.pcu = None
 223
 224         else:
 225                 agg.pcu_status = "nopcu"
 226                 agg.pcu_short_status = "none"
 227                 agg.pcu = None
 228
 229
 230         if node.kernel_version:
 231                 agg.kernel = node.kernel_version.split()[2]
 232         else:
 233                 agg.kernel = ""
 234
 235         try:
 236                 agg.loginbase = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 237         except:
 238                 agg.loginbase = "unknown"
 239
 240         if agg.loginbase:
 241                 agg.site = HistorySiteRecord.by_loginbase(agg.loginbase)
 242
 243                 if asofdate:
 244                         agg.site = agg.site.get_as_of(asofdate)
 245
 246                 if agg.site is None:
 247                         # TODO: need a cleaner fix for this...
 248                         agg.site = HistorySiteRecord.by_loginbase("pl")
 249                         if not agg.site:
 250                                 agg.site = HistorySiteRecord.by_loginbase("ple")
 251
 252         agg.history = HistoryNodeRecord.by_hostname(node.hostname)
 253         if asofdate:
 254                 agg.history = agg.history.get_as_of(asofdate)
 255
 256         agg.ports = format_ports(node.port_status)
 257
 258         try:
 259                 exists = node.plc_node_stats['last_contact']
 260         except:
 261                 # TODO: this should not assign to the fb object!
 262                 node.plc_node_stats = {'last_contact' : None}
 263
 264         return agg
 265
 266
 267 class Root(controllers.RootController, MonitorXmlrpcServer):
 268         @expose(template="monitorweb.templates.welcome")
 269         def index(self):
 270                 # log.debug("Happy TurboGears Controller Responding For Duty")
 271                 flash("Welcome To MyOps!")
 272                 return dict(now=time.ctime())
 273
 274         @expose(template="monitorweb.templates.nodelist", allow_json=True)
 275         def node3(self, filter=None):
 276                 nhquery = HistoryNodeRecord.query.all()
 277                 query = []
 278                 for nh in nhquery:
 279                         if filter:
 280                                 if nh.status == filter:
 281                                         query.append(nh)
 282                         else:
 283                                 query.append(nh)
 284
 285                 rquery=[]
 286                 for q in query:
 287                         fb = FindbadNodeRecord.get_latest_by(hostname=q.hostname)
 288                         rquery.append(fb)
 289
 290                 return dict(now=time.ctime(), query=rquery)
 291
 292         def node_query(self, filter):
 293                 nhquery = HistoryNodeRecord.query.all()
 294                 query = []
 295                 for nh in nhquery:
 296                         if filter:
 297                                 if nh.status == filter:
 298                                         query.append(nh)
 299                         else:
 300                                 query.append(nh)
 301
 302                 rquery=[]
 303                 for q in query:
 304                         fb = FindbadNodeRecord.get_latest_by(hostname=q.hostname)
 305                         agg = prep_node_for_display(fb)
 306                         rquery.append(agg)
 307                 return rquery
 308
 309         @expose("cheetah:monitorweb.templates.nodelist_plain", as_format="plain",
 310                 accept_format="text/plain", content_type="text/plain")
 311         @expose(template="monitorweb.templates.nodelist", allow_json=True)
 312         def node2(self, filter=None):
 313                 rquery=self.node_query(filter)
 314                 widget = NodeWidget(template='monitorweb.templates.node_template')
 315                 return dict(now=time.ctime(), query=rquery, nodewidget=widget)
 316
 317         @expose("cheetah:monitorweb.templates.query_plain", as_format="plain",
 318                 accept_format="text/plain", content_type="text/plain")
 319         @expose(template="monitorweb.templates.query", allow_json=True)
 320         def query(self, **data):
 321                 query = []
 322
 323                 for k in data:
 324                         print k, data[k]
 325
 326                 fbquery = None
 327
 328                 if 'object' in data and data['object'] == "nodes":
 329                         fbquery = FindbadNodeRecord.get_all_latest()
 330                 elif 'object' in data and data['object'] == "nodehistory":
 331                         hostname = data['nodehistory_hostname']
 332                         data['date_checked'] = 'date_checked'
 333                         fbrecord = FindbadNodeRecord.get_by(hostname=hostname)
 334                         fbquery = fbrecord.versions[-500:]
 335
 336                 if fbquery:
 337                         for node in fbquery:
 338                                 # NOTE: reformat some fields.
 339                                 if type(node) is not type(FindbadNodeRecord):
 340                                         agg = node.__dict__.copy()
 341                                 else:
 342                                         agg = node.to_dict()
 343                                 agg.update(agg['plc_node_stats'])
 344                                 if agg['install_date']:
 345                                         agg['install_date'] = time.mktime(time.strptime(agg['install_date'], "%a %b %d %H:%M:%S %Y"))
 346                                 if agg['kernel_version']:
 347                                         agg['kernel_version'] = agg['kernel_version'].split()[2]
 348                                 if 'traceroute' in data and agg['traceroute']:
 349                                         agg['traceroute'] = "<pre>" + agg['traceroute'] + "</pre>"
 350                                 if 'rpmvalue' in data and 'rpms' in data:
 351                                         if agg['rpms']:
 352                                                 rpm_list = agg['rpms'].split()
 353                                                 rpm_list = filter(lambda x: data['rpmvalue'] in x, rpm_list)
 354                                                 agg['rpms'] = " ".join(rpm_list)
 355
 356                                 query.append(agg)
 357
 358                 fields=data.copy()
 359
 360                 try:
 361                         del fields['object']
 362                         del fields['rpmvalue']
 363                         del fields['nodehistory_hostname']
 364                 except: pass
 365                 return dict(now=time.ctime(), query=query, fields=fields, data=data, queryform=getNodeQueryForm())
 366
 367         @expose(template="monitorweb.templates.nodefast", allow_json=True)
 368         def node(self, filter=None):
 369                 nhquery = HistoryNodeRecord.query.all()
 370                 query = []
 371                 for nh in nhquery:
 372                         if filter:
 373                                 if nh.status == filter:
 374                                         agg = prep_nodehist(nh)
 375                                         query.append(agg)
 376                         else:
 377                                 agg = prep_nodehist(nh)
 378                                 query.append(agg)
 379
 380                 return dict(now=time.ctime(), query=query)
 381
 382         @expose(template="monitorweb.templates.nodelist")
 383         def nodeslow(self, filter='boot'):
 384                 print "NODE------------------"
 385                 print "befor-len: ", len( [ i for i in session] )
 386                 session.flush(); session.clear()
 387                 print "after-len: ", len( [ i for i in session] )
 388                 fbquery = FindbadNodeRecord.get_all_latest()
 389                 query = []
 390                 filtercount = {'down' : 0, 'boot': 0, 'debug' : 0, 'diagnose' : 0, 'disabled': 0,
 391                                                 'neverboot' : 0, 'pending' : 0, 'all' : 0, None : 0}
 392                 for node in fbquery:
 393                         # NOTE: reformat some fields.
 394                         agg = prep_node_for_display(node)
 395
 396                         if not agg.history:
 397                                 continue
 398
 399                         if agg.history.status in ['down', 'offline']:
 400                                 if node.plc_node_stats and node.plc_node_stats['last_contact'] != None:
 401                                         filtercount['down'] += 1
 402                                 else:
 403                                         filtercount['neverboot'] += 1
 404                         elif agg.history.status in ['good', 'online']:
 405                                 filtercount['boot'] += 1
 406                         elif agg.history.status in ['debug', 'monitordebug']:
 407                                 filtercount['debug'] += 1
 408                         else:
 409                                 if filtercount.has_key(agg.history.status):
 410                                         filtercount[agg.history.status] += 1
 411
 412
 413                         # NOTE: apply filter
 414                         if filter == "neverboot":
 415                                 if not node.plc_node_stats or node.plc_node_stats['last_contact'] == None:
 416                                         query.append(agg)
 417                         elif filter == "all":
 418                                 query.append(agg)
 419                         elif filter == agg.history.status:
 420                                 query.append(agg)
 421                         elif filter == 'boot':
 422                                 query.append(agg)
 423
 424
 425                 widget = NodeWidget(template='monitorweb.templates.node_template')
 426                 return dict(now=time.ctime(), query=query, fc=filtercount, nodewidget=widget)
 427
 428         def nodeaction_handler(self, tg_exceptions=None):
 429                 """Handle any kind of error."""
 430                 print "NODEACTION_HANDLER------------------"
 431
 432                 if 'pcuid' in request.params:
 433                         pcuid = request.params['pcuid']
 434                 else:
 435                         refurl = request.headers.get("Referer",link("pcu"))
 436                         print refurl
 437
 438                         # TODO: do this more intelligently...
 439                         uri_fields = urllib.splitquery(refurl)
 440                         if uri_fields[1] is not None:
 441                                 val = query_to_dict(uri_fields[1])
 442                                 if 'pcuid' in val:
 443                                         pcuid = val['pcuid']
 444                                 elif 'hostname' in val:
 445                                         pcuid = FindbadNodeRecord.get_latest_by(hostname=val['hostname']).plc_pcuid
 446                                 else:
 447                                         pcuid=None
 448                         else:
 449                                 pcuid=None
 450
 451                 cherry_trail = cherrypy._cputil.get_object_trail()
 452                 for i in cherry_trail:
 453                         print "trail: ", i
 454
 455                 print pcuid
 456                 return self.pcuview(None, pcuid, **dict(exceptions=tg_exceptions))
 457
 458         def nodeaction(self, **data):
 459                 print "NODEACTION------------------"
 460                 for item in data.keys():
 461                         print "%s %s" % ( item, data[item] )
 462
 463                 if 'hostname' in data:
 464                         hostname = data['hostname']
 465                 else:
 466                         flash("No hostname given in submitted data")
 467                         return
 468
 469                 if 'submit' in data or 'type' in data:
 470                         try:
 471                                 action = data['submit']
 472                         except:
 473                                 action = data['type']
 474                 else:
 475                         flash("No submit action given in submitted data")
 476                         return
 477
 478                 if action == "Reboot":
 479                         print "REBOOT: %s" % hostname
 480                         ret = reboot.reboot_str(str(hostname))
 481                         print ret
 482                         if ret: raise RuntimeError("Error using PCU: " + str(ret))
 483                         flash("Reboot appeared to work.  Allow at most 5 minutes.  Then run ExternalScan to check current status.")
 484
 485                 elif action == "ExternalScan":
 486                         scanapi.externalprobe(str(hostname))
 487                         flash("External Scan Successful!")
 488                 elif action == "InternalScan":
 489                         scanapi.internalprobe(str(hostname))
 490                         flash("Internal Scan Successful!")
 491                 else:
 492                         # unknown action
 493                         raise RuntimeError("Unknown action given")
 494                 return
 495
 496         @expose(template="monitorweb.templates.simpleview")
 497         def simpleview(self, **data):
 498                 return self.pre_view(**data)
 499
 500         @expose(template="monitorweb.templates.simpleview")
 501         def pcuview(self, **data):
 502                 return self.pre_view(**data)
 503
 504         @expose(template="monitorweb.templates.detailview")
 505         def detailview(self, **data):
 506                 return self.pre_view(**data)
 507
 508
 509         def pre_view(self, **data):
 510                 session.flush(); session.clear()
 511
 512                 loginbase=None
 513                 loginbase_list=[]
 514                 hostname=None
 515                 pcuid=None
 516                 since=20
 517                 # if objtype is not None, then treat 'hostname' or 'loginbase' as a search pattern
 518                 objtype=None
 519
 520                 exceptions = None
 521                 sitequery=[]
 522                 nodequery=[]
 523                 pcuquery=[]
 524                 actions=[]
 525                 actions_list=[]
 526
 527                 for key in data:
 528                         print key, data[key]
 529
 530                 if 'query' in data:
 531                         obj = data['query']
 532                         fields = obj.split(":")
 533                         if len(fields) > 1:
 534                                 objtype = fields[0]
 535                                 obj = fields[1].replace("*", "%")
 536                                 print "obj: %s"% obj
 537
 538                         if len(obj.split(".")) > 1 or objtype == "node":
 539                                 hostname = obj
 540                         else:
 541                                 loginbase = obj
 542
 543                 if 'loginbase' in data:
 544                         loginbase = data['loginbase']
 545
 546                 if 'hostname' in data:
 547                         hostname = data['hostname']
 548
 549                 if 'pcuid' in data:
 550                         try: pcuid = int(data['pcuid'])
 551                         except: pcuid = None
 552
 553                 if 'since' in data:
 554                         try: since = int(since)
 555                         except: since = 20
 556
 557                 if pcuid:
 558                         print "pcuid: %s" % pcuid
 559                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=pcuid)
 560                         loginbase_list += [ PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base'] ]
 561
 562                 if hostname:
 563                         if not objtype:
 564                                 nodes = [ FindbadNodeRecord.get_latest_by(hostname=hostname) ]
 565                         else:
 566                                 nodes = FindbadNodeRecord.query.filter(FindbadNodeRecord.hostname.like(hostname))
 567
 568                         for node in nodes:
 569                                 lb = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 570                                 if lb not in loginbase_list:
 571                                         loginbase_list += [ lb ]
 572
 573                 if loginbase:
 574                         if not objtype:
 575                                 loginbase_list = [ loginbase ]
 576                         else:
 577                                 loginbase_list = HistorySiteRecord.query.filter(HistorySiteRecord.loginbase.like(loginbase))
 578                                 loginbase_list = [ l.loginbase for l in loginbase_list ]
 579
 580
 581                 if loginbase_list:
 582                         for loginbase in loginbase_list:
 583                                 actions = ActionRecord.query.filter_by(loginbase=loginbase
 584                                                                 ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since)
 585                                                                 ).order_by(ActionRecord.date_created.desc())
 586                                 actions_list += [ a for a in actions ]
 587                                 site = HistorySiteRecord.by_loginbase(loginbase)
 588                                 if site:
 589                                         sitequery.append(site)
 590                                 # NOTE: because a single pcu may be assigned to multiple hosts,
 591                                 # track unique pcus by their plc_pcuid, then turn dict into list
 592                                 pcus = {}
 593                                 for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 594                                                 # NOTE: reformat some fields.
 595                                                 agg = prep_node_for_display(node)
 596                                                 nodequery += [agg]
 597                                                 if agg.pcu:
 598                                                         pcus[agg.pcu.pcu.plc_pcuid] = agg.pcu
 599
 600                                 for pcuid_key in pcus:
 601                                         pcuquery += [pcus[pcuid_key]]
 602
 603                 actionlist_widget = ActionListWidget(template='monitorweb.templates.actionlist_template')
 604                 return dict(sitequery=sitequery, pcuquery=pcuquery, nodequery=nodequery, actions=actions_list, actionlist_widget=actionlist_widget, since=since, exceptions=exceptions)
 605
 606
 607         # TODO: add form validation
 608         @expose(template="monitorweb.templates.pcuview")
 609         @exception_handler(nodeaction_handler,"isinstance(tg_exceptions,RuntimeError)")
 610         def pcuviewold(self, loginbase=None, pcuid=None, hostname=None, since=20, **data):
 611                 session.flush(); session.clear()
 612                 sitequery=[]
 613                 pcuquery=[]
 614                 nodequery=[]
 615                 actions=[]
 616                 exceptions = None
 617
 618                 try: since = int(since)
 619                 except: since = 7
 620
 621                 for key in data:
 622                         print key, data[key]
 623
 624                 if 'submit' in data.keys() or 'type' in data.keys():
 625                         if hostname: data['hostname'] = hostname
 626                         self.nodeaction(**data)
 627                 if 'exceptions' in data:
 628                         exceptions = data['exceptions']
 629
 630                 if 'query' in data:
 631                         obj = data['query']
 632                         if len(obj.split(".")) > 1: hostname = obj
 633                         else: loginbase=obj
 634
 635                 if pcuid:
 636                         print "pcuid: %s" % pcuid
 637                         pcu = FindbadPCURecord.get_latest_by(plc_pcuid=pcuid)
 638                         loginbase = PlcSite.query.get(pcu.plc_pcu_stats['site_id']).plc_site_stats['login_base']
 639
 640                 if hostname:
 641                         node = FindbadNodeRecord.get_latest_by(hostname=hostname)
 642                         loginbase = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base']
 643
 644                 if loginbase:
 645                         actions = ActionRecord.query.filter_by(loginbase=loginbase
 646                                                         ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since)
 647                                                         ).order_by(ActionRecord.date_created.desc())
 648                         actions = [ a for a in actions ]
 649                         sitequery = [HistorySiteRecord.by_loginbase(loginbase)]
 650                         pcus = {}
 651                         for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 652                                         # NOTE: reformat some fields.
 653                                         agg = prep_node_for_display(node)
 654                                         nodequery += [agg]
 655                                         if agg.pcu: #.pcu.plc_pcuid:    # not None
 656                                                 #pcu = FindbadPCURecord.get_latest_by(plc_pcuid=agg.plc_pcuid)
 657                                                 #prep_pcu_for_display(pcu)
 658                                                 pcus[agg.pcu.pcu.plc_pcuid] = agg.pcu
 659
 660                         for pcuid_key in pcus:
 661                                 pcuquery += [pcus[pcuid_key]]
 662
 663                 return dict(sitequery=sitequery, pcuquery=pcuquery, nodequery=nodequery, actions=actions, since=since, exceptions=exceptions)
 664
 665         @expose(template="monitorweb.templates.pcuhistory")
 666         def pcuhistory(self, pcu_id=None):
 667                 query = []
 668                 if pcu_id:
 669                         fbnode = HistoryPCURecord.get_by(plc_pcuid=pcu_id)
 670                         l = fbnode.versions[-100:]
 671                         l.reverse()
 672                         for pcu in l:
 673                                 #prep_node_for_display(node)
 674                                 query.append(pcu)
 675
 676                 return dict(query=query, pcu_id=pcu_id)
 677
 678         @expose(template="monitorweb.templates.nodescanhistory")
 679         def nodescanhistory(self, hostname=None, length=10):
 680                 try: length = int(length)
 681                 except: length = 21
 682
 683                 fbnode = FindbadNodeRecord.get_by(hostname=hostname)
 684                 # TODO: add links for earlier history if desired.
 685                 l = fbnode.versions[-length:]
 686                 l.reverse()
 687                 query=[]
 688                 for node in l:
 689                         agg = prep_node_for_display(node, pcuhash=None, preppcu=False, asofdate=node.timestamp)
 690                         query.append(agg)
 691
 692                 if 'length' in request.params:
 693                         del request.params['length']
 694                 return dict(query=query, hostname=hostname, params=request.params)
 695
 696         @expose(template="monitorweb.templates.nodehistory")
 697         def nodehistory(self, hostname=None):
 698                 query = []
 699                 if hostname:
 700                         fbnode = HistoryNodeRecord.get_by(hostname=hostname)
 701                         l = fbnode.versions[-100:]
 702                         l.reverse()
 703                         for node in l:
 704                                 #prep_node_for_display(node)
 705                                 query.append(node)
 706
 707                 return dict(query=query, hostname=hostname)
 708
 709         @expose(template="monitorweb.templates.sitehistory")
 710         def sitehistory(self, loginbase=None):
 711                 query = []
 712                 if loginbase:
 713                         fbsite = HistorySiteRecord.get_by(loginbase=loginbase)
 714                         # TODO: add links for earlier history if desired.
 715                         l = fbsite.versions[-100:]
 716                         l.reverse()
 717                         for site in l:
 718                                 query.append(site)
 719                 return dict(query=query, loginbase=loginbase)
 720
 721
 722         @expose(template="monitorweb.templates.pculist")
 723         def pcu(self, filter='all'):
 724                 print "PCUVIEW------------------"
 725                 print "befor-len: ", len( [ i for i in session] )
 726                 session.flush(); session.clear()
 727                 print "after-len: ", len( [ i for i in session] )
 728                 fbquery = FindbadPCURecord.get_all_latest()
 729                 query = []
 730                 filtercount = {'ok' : 0, 'NetDown': 0, 'Not_Run' : 0, 'pending' : 0, 'all' : 0}
 731                 for node in fbquery:
 732
 733                         # NOTE: count filter
 734                         if node.reboot_trial_status == str(0):
 735                                 filtercount['ok'] += 1
 736                         elif node.reboot_trial_status == 'NetDown' or node.reboot_trial_status == 'Not_Run':
 737                                 filtercount[node.reboot_trial_status] += 1
 738                         else:
 739                                 filtercount['pending'] += 1
 740
 741                         pcuagg = prep_pcu_for_display(node)
 742
 743                         # NOTE: apply filter
 744                         if filter == "all":
 745                                 query.append(pcuagg)
 746                         elif filter == "ok" and node.reboot_trial_status == str(0):
 747                                 query.append(pcuagg)
 748                         elif filter == node.reboot_trial_status:
 749                                 query.append(pcuagg)
 750                         elif filter == "pending":
 751                                 # TODO: look in message logs...
 752                                 if node.reboot_trial_status != str(0) and \
 753                                         node.reboot_trial_status != 'NetDown' and \
 754                                         node.reboot_trial_status != 'Not_Run':
 755
 756                                         query.append(pcuagg)
 757
 758                 return dict(query=query, fc=filtercount)
 759
 760         @expose(template="monitorweb.templates.sitelist")
 761         def site(self, filter='all'):
 762                 print "SITE------------------"
 763                 print "befor-len: ", len( [ i for i in session] )
 764                 session.flush(); session.clear()
 765                 print "after-len: ", len( [ i for i in session] )
 766                 filtercount = {'good' : 0, 'down': 0, 'online':0, 'offline' : 0, 'new' : 0, 'pending' : 0, 'all' : 0}
 767                 fbquery = HistorySiteRecord.query.all()
 768                 query = []
 769                 for site in fbquery:
 770                         # count filter
 771                         filtercount['all'] += 1
 772                         if site.new and site.slices_used == 0 and not site.enabled:
 773                                 filtercount['new'] += 1
 774                         elif not site.enabled:
 775                                 filtercount['pending'] += 1
 776                         elif site.status in ['good', 'online']:
 777                                 filtercount['good'] += 1
 778                         elif site.status in ['down', 'offline']:
 779                                 filtercount['down'] += 1
 780
 781                         # apply filter
 782                         if filter == "all":
 783                                 query.append(site)
 784                         elif filter == 'new' and site.new and site.slices_used == 0 and not site.enabled:
 785                                 query.append(site)
 786                         elif filter == "pending" and not site.enabled:
 787                                 query.append(site)
 788                         elif filter == 'good' and site.status in ['good', 'online']:
 789                                 query.append(site)
 790                         elif filter == 'down' and site.status in ['down', 'offline']:
 791                                 query.append(site)
 792
 793                 return dict(query=query, fc=filtercount)
 794         @expose(template="monitorweb.templates.sitesummary")
 795         def sitesummary(self, loginbase="princeton"):
 796                 nodequery = []
 797                 for node in FindbadNodeRecord.query.filter_by(loginbase=loginbase):
 798                         agg = prep_node_for_display(node)
 799                         nodequery += [agg]
 800
 801                 return dict(nodequery=nodequery, loginbase=loginbase)
 802
 803         @expose(template="monitorweb.templates.summary")
 804         def summary(self, since=7):
 805                 sumdata = {}
 806                 sumdata['nodes'] = {}
 807                 sumdata['sites'] = {}
 808                 sumdata['pcus'] = {}
 809
 810                 def summarize(query, type):
 811                         for o in query:
 812                                 if o.status not in sumdata[type]:
 813                                         sumdata[type][o.status] = 0
 814                                 sumdata[type][o.status] += 1
 815
 816                 fbquery = HistorySiteRecord.query.all()
 817                 summarize(fbquery, 'sites')
 818                 fbquery = HistoryPCURecord.query.all()
 819                 summarize(fbquery, 'pcus')
 820                 fbquery = HistoryNodeRecord.query.all()
 821                 summarize(fbquery, 'nodes')
 822
 823                 if 'monitordebug' in sumdata['nodes']:
 824                         d = sumdata['nodes']['monitordebug']
 825                         del sumdata['nodes']['monitordebug']
 826                         sumdata['nodes']['failboot'] = d
 827
 828                 return dict(sumdata=sumdata, setorder=['good', 'offline', 'down', 'online'])
 829
 830         @expose(template="monitorweb.templates.actionsummary")
 831         def actionsummary(self, since=7):
 832                 from monitor.wrapper.emailTxt import mailtxt
 833
 834                 types = filter(lambda x: 'notice' in x, dir(mailtxt))
 835                 results = {}
 836
 837                 print mon_metadata.bind
 838                 if session.bind is None:
 839                         #TODO: figure out why this value gets cleared out...
 840                         session.bind = mon_metadata.bind
 841                 result = session.execute("select distinct(action_type) from actionrecord;")
 842
 843                 types = [r[0] for r in result]
 844
 845                 try: since = int(since)
 846                 except: since = 7
 847
 848                 for  t in types:
 849                         acts = ActionRecord.query.filter(ActionRecord.action_type==t
 850                                         ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since))
 851                         results[t] = acts.count()
 852                 return dict(results=results)
 853
 854         @expose(template="monitorweb.templates.actionlist")
 855         def actionlist(self, since=7, action_type=None, loginbase=None):
 856
 857                 try: since = int(since)
 858                 except: since = 7
 859
 860                 acts_query = ActionRecord.query.filter(
 861                                           ActionRecord.date_created >= datetime.now() - timedelta(since)
 862                                          )
 863                 if loginbase:
 864                         acts_query = acts_query.filter_by(loginbase=loginbase)
 865
 866                 if action_type:
 867                         acts_query = acts_query.filter(ActionRecord.action_type==action_type)
 868
 869                 acts = acts_query.order_by(ActionRecord.date_created.desc())
 870
 871                 query = [ a for a in acts ]
 872
 873                 return dict(actions=query, action_type=action_type, since=since)
 874
 875         @cherrypy.expose()
 876         def upload(self, log, **keywords):
 877                 hostname = None
 878                 logtype = None
 879                 logtype_list = ['bm.log', ]
 880
 881                 if 'hostname' in keywords:
 882                         hostname = keywords['hostname']
 883                 if 'type' in keywords and keywords['type'] in logtype_list:
 884                         logtype = keywords['type']
 885
 886                 if not hostname: return ""
 887                 if not logtype: return "unknown logtype: %s" % logtype
 888
 889                 short_target_filename = bootman.bootmanager_log_name(hostname)
 890                 abs_target_filename = os.path.join(config.MONITOR_BOOTMANAGER_LOG, short_target_filename)
 891                 print "write data: %s" % abs_target_filename
 892                 util.file.dumpFile(abs_target_filename, log.file.read())
 893                 bootman.bootmanager_log_action(hostname, short_target_filename, logtype)
 894                 session.flush()
 895
 896                 print "redirecting 3"
 897
 898                 return dict()