nodequery.py

   1 #!/usr/bin/python
   2
   3
   4 import sys
   5 from monitor import database
   6 from nodecommon import *
   7 from monitor.model import Record
   8 import glob
   9 import os
  10 import traceback
  11
  12 import time
  13 import re
  14 import string
  15
  16 from monitor.pcu import reboot
  17 from monitor.wrapper import plc, plccache
  18 api = plc.getAuthAPI()
  19
  20 from monitor.database.info.model import FindbadNodeRecordSync, FindbadNodeRecord, session
  21 from monitor import util
  22 from monitor import config
  23
  24
  25 class NoKeyException(Exception): pass
  26
  27 def daysdown_print_nodeinfo(fbnode, hostname):
  28         fbnode['hostname'] = hostname
  29         fbnode['daysdown'] = Record.getStrDaysDown(fbnode)
  30         fbnode['intdaysdown'] = Record.getDaysDown(fbnode)
  31
  32         print "%(intdaysdown)5s %(hostname)-44s | %(state)10.10s | %(daysdown)s" % fbnode
  33
  34 def fb_print_nodeinfo(fbnode, hostname, fields=None):
  35         #fbnode['hostname'] = hostname
  36         #fbnode['checked'] = diff_time(fbnode['checked'])
  37         if fbnode['bootcd_version']:
  38                 fbnode['bootcd_version'] = fbnode['bootcd_version'].split()[-1]
  39         else:
  40                 fbnode['bootcd_version'] = "unknown"
  41         fbnode['pcu'] = color_pcu_state(fbnode)
  42
  43         if not fields:
  44                 if ( fbnode['observed_status'] is not None and \
  45                    'DOWN' in fbnode['observed_status'] ) or \
  46                    fbnode['kernel_version'] is None:
  47                         fbnode['kernel_version'] = ""
  48                 else:
  49                         fbnode['kernel_version'] = fbnode['kernel_version'].split()[2]
  50
  51                 if fbnode['plc_node_stats'] is not None:
  52                         fbnode['boot_state'] = fbnode['plc_node_stats']['boot_state']
  53                 else:
  54                         fbnode['boot_state'] = "unknown"
  55
  56                 try:
  57                         if len(fbnode['nodegroups']) > 0:
  58                                 fbnode['category'] = fbnode['nodegroups'][0]
  59                 except:
  60                         #print "ERROR!!!!!!!!!!!!!!!!!!!!!"
  61                         pass
  62
  63                 print "%(hostname)-45s | %(date_checked)11.11s | %(boot_state)5.5s| %(observed_status)8.8s | %(ssh_status)5.5s | %(pcu)6.6s | %(bootcd_version)6.6s | %(kernel_version)s" % fbnode
  64         else:
  65                 format = ""
  66                 for f in fields:
  67                         format += "%%(%s)s " % f
  68                 print format % fbnode
  69
  70 def first(path):
  71         indexes = path.split(".")
  72         return indexes[0]
  73
  74 def get(fb, path):
  75     indexes = path.split(".")
  76     values = fb
  77     for index in indexes:
  78         if index in values:
  79             values = values[index]
  80         else:
  81             raise NoKeyException(index)
  82     return values
  83
  84 def verifyType(constraints, data):
  85         """
  86                 constraints is a list of key, value pairs.
  87                 # [ {... : ...}==AND , ... , ... , ] == OR
  88         """
  89         con_or_true = False
  90         for con in constraints:
  91                 #print "con: %s" % con
  92                 if len(con.keys()) == 0:
  93                         con_and_true = False
  94                 else:
  95                         con_and_true = True
  96
  97                 for key in con.keys():
  98                         #print "looking at key: %s" % key
  99                         if data is None:
 100                                 con_and_true = False
 101                                 break
 102
 103                         try:
 104                                 get(data,key)
 105                                 o = con[key]
 106                                 if o.name() == "Match":
 107                                         if get(data,key) is not None:
 108                                                 value_re = re.compile(o.value)
 109                                                 con_and_true = con_and_true & (value_re.search(get(data,key)) is not None)
 110                                         else:
 111                                                 con_and_true = False
 112                                 elif o.name() == "ListMatch":
 113                                         if get(data,key) is not None:
 114                                                 match = False
 115                                                 for listitem in get(data,key):
 116                                                         value_re = re.compile(o.value)
 117                                                         if value_re.search(listitem) is not None:
 118                                                                 match = True
 119                                                                 break
 120                                                 con_and_true = con_and_true & match
 121                                         else:
 122                                                 con_and_true = False
 123                                 elif o.name() == "Is":
 124                                         con_and_true = con_and_true & (get(data,key) == o.value)
 125                                 elif o.name() == "FilledIn":
 126                                         con_and_true = con_and_true & (len(get(data,key)) > 0)
 127                                 elif o.name() == "PortOpen":
 128                                         if get(data,key) is not None:
 129                                                 v = get(data,key)
 130                                                 con_and_true = con_and_true & (v[str(o.value)] == "open")
 131                                         else:
 132                                                 con_and_true = False
 133                                 else:
 134                                         value_re = re.compile(o.value)
 135                                         con_and_true = con_and_true & (value_re.search(get(data,key)) is not None)
 136
 137                         except NoKeyException, key:
 138                                 print "missing key %s" % key,
 139                                 pass
 140                                 #print "missing key %s" % key
 141                                 #con_and_true = False
 142
 143                 con_or_true = con_or_true | con_and_true
 144
 145         return con_or_true
 146
 147 def verifyDBrecord(constraints, record):
 148         """
 149                 constraints is a list of key, value pairs.
 150                 # [ {... : ...}==AND , ... , ... , ] == OR
 151         """
 152         def has_key(obj, key):
 153                 try:
 154                         x = obj.__getattribute__(key)
 155                         return True
 156                 except:
 157                         return False
 158
 159         def get_val(obj, key):
 160                 try:
 161                         return obj.__getattribute__(key)
 162                 except:
 163                         return None
 164
 165         def get(obj, path):
 166                 indexes = path.split("/")
 167                 value = get_val(obj,indexes[0])
 168                 if value is not None and len(indexes) > 1:
 169                         for key in indexes[1:]:
 170                                 if key in value:
 171                                         value = value[key]
 172                                 else:
 173                                         raise NoKeyException(key)
 174                 return value
 175
 176         #print constraints, record
 177
 178         con_or_true = False
 179         for con in constraints:
 180                 #print "con: %s" % con
 181                 if len(con.keys()) == 0:
 182                         con_and_true = False
 183                 else:
 184                         con_and_true = True
 185
 186                 for key in con.keys():
 187                         #print "looking at key: %s" % key
 188                         if has_key(record, key):
 189                                 value_re = re.compile(con[key])
 190                                 if type([]) == type(get(record,key)):
 191                                         local_or_true = False
 192                                         for val in get(record,key):
 193                                                 local_or_true = local_or_true | (value_re.search(val) is not None)
 194                                         con_and_true = con_and_true & local_or_true
 195                                 else:
 196                                         if get(record,key) is not None:
 197                                                 con_and_true = con_and_true & (value_re.search(get(record,key)) is not None)
 198                         else:
 199                                 print "missing key %s" % key,
 200                                 pass
 201
 202                 con_or_true = con_or_true | con_and_true
 203
 204         return con_or_true
 205
 206 def verify(constraints, data):
 207         """
 208                 constraints is a list of key, value pairs.
 209                 # [ {... : ...}==AND , ... , ... , ] == OR
 210         """
 211         con_or_true = False
 212         for con in constraints:
 213                 #print "con: %s" % con
 214                 if len(con.keys()) == 0:
 215                         con_and_true = False
 216                 else:
 217                         con_and_true = True
 218
 219                 for key in con.keys():
 220                         #print "looking at key: %s" % key
 221                         if first(key) in data:
 222                                 value_re = re.compile(con[key])
 223                                 if type([]) == type(get(data,key)):
 224                                         local_or_true = False
 225                                         for val in get(data,key):
 226                                                 local_or_true = local_or_true | (value_re.search(val) is not None)
 227                                         con_and_true = con_and_true & local_or_true
 228                                 else:
 229                                         if get(data,key) is not None:
 230                                                 con_and_true = con_and_true & (value_re.search(get(data,key)) is not None)
 231                         elif first(key) not in data:
 232                                 print "missing key %s" % first(key)
 233
 234                 con_or_true = con_or_true | con_and_true
 235
 236         return con_or_true
 237
 238 def query_to_dict(query):
 239
 240         ad = []
 241
 242         or_queries = query.split('||')
 243         for or_query in or_queries:
 244                 and_queries = or_query.split('&&')
 245
 246                 d = {}
 247
 248                 for and_query in and_queries:
 249                         (key, value) = and_query.split('=')
 250                         d[key] = value
 251
 252                 ad.append(d)
 253
 254         return ad
 255
 256 def pcu_in(fbdata):
 257         if 'plcnode' in fbdata:
 258                 if 'pcu_ids' in fbdata['plcnode']:
 259                         if len(fbdata['plcnode']['pcu_ids']) > 0:
 260                                 return True
 261         return False
 262
 263 def pcu_select(str_query, nodelist=None):
 264         pcunames = []
 265         nodenames = []
 266         if str_query is None: return (nodenames, pcunames)
 267
 268         if True:
 269                 fbquery = FindbadNodeRecord.get_all_latest()
 270                 fb_nodelist = [ n.hostname for n in fbquery ]
 271         if True:
 272                 fbpcuquery = FindbadPCURecord.get_all_latest()
 273                 fbpcu_list = [ p.plc_pcuid for p in fbpcuquery ]
 274
 275         dict_query = query_to_dict(str_query)
 276
 277         for noderec in fbquery:
 278                 if nodelist is not None:
 279                         if noderec.hostname not in nodelist: continue
 280
 281                 fb_nodeinfo  = noderec.to_dict()
 282                 if pcu_in(fb_nodeinfo):
 283                         pcurec = FindbadPCURecord.get_latest_by(plc_pcuid=get(fb_nodeinfo, 'plc_node_stats.pcu_ids')[0])
 284                         pcuinfo = pcurec.to_dict()
 285                         if verify(dict_query, pcuinfo):
 286                                 nodenames.append(noderec.hostname)
 287                                 str = "cmdhttps/locfg.pl -s %s -f iloxml/License.xml -u %s -p '%s' | grep MESSAGE" % \
 288                                                         (reboot.pcu_name(pcuinfo), pcuinfo['username'], pcuinfo['password'])
 289                                 pcunames.append(pcuinfo['plc_pcuid'])
 290         return (nodenames, pcunames)
 291
 292 def node_select(str_query, nodelist=None, fb=None):
 293
 294         hostnames = []
 295         if str_query is None: return hostnames
 296
 297         #print str_query
 298         dict_query = query_to_dict(str_query)
 299         #print dict_query
 300
 301         for node in nodelist:
 302                 #if nodelist is not None:
 303                 #       if node not in nodelist: continue
 304
 305                 try:
 306                         fb_noderec = None
 307                         #fb_noderec = FindbadNodeRecord.query.filter(FindbadNodeRecord.hostname==node).order_by(FindbadNodeRecord.date_checked.desc()).first()
 308                         fb_noderec = FindbadNodeRecord.get_latest_by(hostname=node)
 309                 except:
 310                         print traceback.print_exc()
 311                         continue
 312
 313                 if fb_noderec:
 314                         fb_nodeinfo = fb_noderec.to_dict()
 315
 316                         #fb_nodeinfo['pcu'] = color_pcu_state(fb_nodeinfo)
 317                         #if 'plcnode' in fb_nodeinfo:
 318                         #       fb_nodeinfo.update(fb_nodeinfo['plcnode'])
 319
 320                         #if verifyDBrecord(dict_query, fb_nodeinfo):
 321                         if verify(dict_query, fb_nodeinfo):
 322                                 #print fb_nodeinfo.keys()
 323                                 #print node #fb_nodeinfo
 324                                 hostnames.append(node)
 325                         else:
 326                                 #print "NO MATCH", node
 327                                 pass
 328
 329         return hostnames
 330
 331
 332 def main():
 333
 334         from monitor import parser as parsermodule
 335         parser = parsermodule.getParser()
 336
 337         parser.set_defaults(node=None, fromtime=None, select=None, list=None, listkeys=False,
 338                                                 pcuselect=None, nodelist=None, daysdown=None, fields=None)
 339         parser.add_option("", "--daysdown", dest="daysdown", action="store_true",
 340                                                 help="List the node state and days down...")
 341         parser.add_option("", "--select", dest="select", metavar="key=value",
 342                                                 help="List all nodes with the given key=value pattern")
 343         parser.add_option("", "--fields", dest="fields", metavar="key,list,...",
 344                                                 help="a list of keys to display for each entry.")
 345         parser.add_option("", "--list", dest="list", action="store_true",
 346                                                 help="Write only the hostnames as output.")
 347         parser.add_option("", "--pcuselect", dest="pcuselect", metavar="key=value",
 348                                                 help="List all nodes with the given key=value pattern")
 349         parser.add_option("", "--nodelist", dest="nodelist", metavar="nodelist.txt",
 350                                                 help="A list of nodes to bring out of debug mode.")
 351         parser.add_option("", "--listkeys", dest="listkeys", action="store_true",
 352                                                 help="A list of nodes to bring out of debug mode.")
 353         parser.add_option("", "--fromtime", dest="fromtime", metavar="YYYY-MM-DD",
 354                                         help="Specify a starting date from which to begin the query.")
 355
 356         parser = parsermodule.getParser(['defaults'], parser)
 357         config = parsermodule.parse_args(parser)
 358
 359         if config.fromtime:
 360                 path = "archive-pdb"
 361                 archive = database.SPickle(path)
 362                 d = datetime_fromstr(config.fromtime)
 363                 glob_str = "%s*.production.findbad.pkl" % d.strftime("%Y-%m-%d")
 364                 os.chdir(path)
 365                 #print glob_str
 366                 file = glob.glob(glob_str)[0]
 367                 #print "loading %s" % file
 368                 os.chdir("..")
 369                 fb = archive.load(file[:-4])
 370         else:
 371                 #fbnodes = FindbadNodeRecord.select(FindbadNodeRecord.q.hostname, orderBy='date_checked',distinct=True).reversed()
 372                 fb = None
 373
 374         #reboot.fb = fbpcu
 375
 376         if config.nodelist:
 377                 nodelist = util.file.getListFromFile(config.nodelist)
 378         else:
 379                 # NOTE: list of nodes should come from findbad db.   Otherwise, we
 380                 # don't know for sure that there's a record in the db..
 381                 plcnodes = plccache.l_nodes
 382                 nodelist = [ node['hostname'] for node in plcnodes ]
 383                 #nodelist = ['planetlab-1.cs.princeton.edu']
 384
 385         pculist = None
 386         if config.select is not None and config.pcuselect is not None:
 387                 nodelist = node_select(config.select, nodelist, fb)
 388                 nodelist, pculist = pcu_select(config.pcuselect, nodelist)
 389         elif config.select is not None:
 390                 nodelist = node_select(config.select, nodelist, fb)
 391         elif config.pcuselect is not None:
 392                 nodelist, pculist = pcu_select(config.pcuselect, nodelist)
 393
 394         if pculist:
 395                 for pcu in pculist:
 396                         print pcu
 397
 398         for node in nodelist:
 399                 config.node = node
 400
 401                 if node not in nodelist:
 402                         continue
 403
 404                 try:
 405                         # Find the most recent record
 406                         fb_noderec = FindbadNodeRecord.query.filter(FindbadNodeRecord.hostname==node).order_by(FindbadNodeRecord.date_checked.desc()).first()
 407                 except:
 408                         print traceback.print_exc()
 409                         pass
 410
 411                 if config.listkeys:
 412                         fb_nodeinfo = fb_noderec.to_dict()
 413                         print "Primary keys available in the findbad object:"
 414                         for key in fb_nodeinfo.keys():
 415                                 print "\t",key
 416                         sys.exit(0)
 417
 418
 419                 if config.list:
 420                         print node
 421                 else:
 422                         if config.daysdown:
 423                                 daysdown_print_nodeinfo(fb_nodeinfo, node)
 424                         else:
 425                                 fb_nodeinfo = fb_noderec.to_dict()
 426                                 if config.select:
 427                                         if config.fields:
 428                                                 fields = config.fields.split(",")
 429                                         else:
 430                                                 fields = None
 431
 432                                         fb_print_nodeinfo(fb_nodeinfo, node, fields)
 433                                 elif not config.select and 'state' in fb_nodeinfo:
 434                                         fb_print_nodeinfo(fb_nodeinfo, node)
 435                                 else:
 436                                         pass
 437
 438 if __name__ == "__main__":
 439         main()