5 from monitor import database
6 from nodecommon import *
7 from unified_model import Record
16 from monitor.pcu import reboot
17 from monitor.wrapper import plc, plccache
18 api = plc.getAuthAPI()
20 from monitor.database.info.model import FindbadNodeRecordSync, FindbadNodeRecord, session
21 from monitor import util
22 from monitor import config
25 class NoKeyException(Exception): pass
27 def daysdown_print_nodeinfo(fbnode, hostname):
28 fbnode['hostname'] = hostname
29 fbnode['daysdown'] = Record.getStrDaysDown(fbnode)
30 fbnode['intdaysdown'] = Record.getDaysDown(fbnode)
32 print "%(intdaysdown)5s %(hostname)-44s | %(state)10.10s | %(daysdown)s" % fbnode
34 def fb_print_nodeinfo(fbnode, hostname, fields=None):
35 #fbnode['hostname'] = hostname
36 #fbnode['checked'] = diff_time(fbnode['checked'])
37 if fbnode['bootcd_version']:
38 fbnode['bootcd_version'] = fbnode['bootcd_version'].split()[-1]
40 fbnode['bootcd_version'] = "unknown"
41 fbnode['pcu'] = color_pcu_state(fbnode)
44 if ( fbnode['observed_status'] is not None and \
45 'DOWN' in fbnode['observed_status'] ) or \
46 fbnode['kernel_version'] is None:
47 fbnode['kernel_version'] = ""
49 fbnode['kernel_version'] = fbnode['kernel_version'].split()[2]
51 if fbnode['plc_node_stats'] is not None:
52 fbnode['boot_state'] = fbnode['plc_node_stats']['boot_state']
54 fbnode['boot_state'] = "unknown"
57 if len(fbnode['nodegroups']) > 0:
58 fbnode['category'] = fbnode['nodegroups'][0]
60 #print "ERROR!!!!!!!!!!!!!!!!!!!!!"
63 print "%(hostname)-45s | %(date_checked)11.11s | %(boot_state)5.5s| %(observed_status)8.8s | %(ssh_status)5.5s | %(pcu)6.6s | %(bootcd_version)6.6s | %(kernel_version)s" % fbnode
67 format += "%%(%s)s " % f
71 indexes = path.split(".")
75 indexes = path.split(".")
79 values = values[index]
81 raise NoKeyException(index)
84 def verifyType(constraints, data):
86 constraints is a list of key, value pairs.
87 # [ {... : ...}==AND , ... , ... , ] == OR
90 for con in constraints:
91 #print "con: %s" % con
92 if len(con.keys()) == 0:
97 for key in con.keys():
98 #print "looking at key: %s" % key
106 if o.name() == "Match":
107 if get(data,key) is not None:
108 value_re = re.compile(o.value)
109 con_and_true = con_and_true & (value_re.search(get(data,key)) is not None)
112 elif o.name() == "ListMatch":
113 if get(data,key) is not None:
115 for listitem in get(data,key):
116 value_re = re.compile(o.value)
117 if value_re.search(listitem) is not None:
120 con_and_true = con_and_true & match
123 elif o.name() == "Is":
124 con_and_true = con_and_true & (get(data,key) == o.value)
125 elif o.name() == "FilledIn":
126 con_and_true = con_and_true & (len(get(data,key)) > 0)
127 elif o.name() == "PortOpen":
128 if get(data,key) is not None:
130 con_and_true = con_and_true & (v[str(o.value)] == "open")
134 value_re = re.compile(o.value)
135 con_and_true = con_and_true & (value_re.search(get(data,key)) is not None)
137 except NoKeyException, key:
138 print "missing key %s" % key,
140 #print "missing key %s" % key
141 #con_and_true = False
143 con_or_true = con_or_true | con_and_true
147 def verifyDBrecord(constraints, record):
149 constraints is a list of key, value pairs.
150 # [ {... : ...}==AND , ... , ... , ] == OR
152 def has_key(obj, key):
154 x = obj.__getattribute__(key)
159 def get_val(obj, key):
161 return obj.__getattribute__(key)
166 indexes = path.split("/")
167 value = get_val(obj,indexes[0])
168 if value is not None and len(indexes) > 1:
169 for key in indexes[1:]:
173 raise NoKeyException(key)
176 #print constraints, record
179 for con in constraints:
180 #print "con: %s" % con
181 if len(con.keys()) == 0:
186 for key in con.keys():
187 #print "looking at key: %s" % key
188 if has_key(record, key):
189 value_re = re.compile(con[key])
190 if type([]) == type(get(record,key)):
191 local_or_true = False
192 for val in get(record,key):
193 local_or_true = local_or_true | (value_re.search(val) is not None)
194 con_and_true = con_and_true & local_or_true
196 if get(record,key) is not None:
197 con_and_true = con_and_true & (value_re.search(get(record,key)) is not None)
199 print "missing key %s" % key,
202 con_or_true = con_or_true | con_and_true
206 def verify(constraints, data):
208 constraints is a list of key, value pairs.
209 # [ {... : ...}==AND , ... , ... , ] == OR
212 for con in constraints:
213 #print "con: %s" % con
214 if len(con.keys()) == 0:
219 for key in con.keys():
220 #print "looking at key: %s" % key
221 if first(key) in data:
222 value_re = re.compile(con[key])
223 if type([]) == type(get(data,key)):
224 local_or_true = False
225 for val in get(data,key):
226 local_or_true = local_or_true | (value_re.search(val) is not None)
227 con_and_true = con_and_true & local_or_true
229 if get(data,key) is not None:
230 con_and_true = con_and_true & (value_re.search(get(data,key)) is not None)
231 elif first(key) not in data:
232 print "missing key %s" % first(key)
234 con_or_true = con_or_true | con_and_true
238 def query_to_dict(query):
242 or_queries = query.split('||')
243 for or_query in or_queries:
244 and_queries = or_query.split('&&')
248 for and_query in and_queries:
249 (key, value) = and_query.split('=')
257 if 'plcnode' in fbdata:
258 if 'pcu_ids' in fbdata['plcnode']:
259 if len(fbdata['plcnode']['pcu_ids']) > 0:
263 def pcu_select(str_query, nodelist=None):
266 if str_query is None: return (nodenames, pcunames)
269 fbquery = FindbadNodeRecord.get_all_latest()
270 fb_nodelist = [ n.hostname for n in fbquery ]
272 fbpcuquery = FindbadPCURecord.get_all_latest()
273 fbpcu_list = [ p.plc_pcuid for p in fbpcuquery ]
275 dict_query = query_to_dict(str_query)
277 for noderec in fbquery:
278 if nodelist is not None:
279 if noderec.hostname not in nodelist: continue
281 fb_nodeinfo = noderec.to_dict()
282 if pcu_in(fb_nodeinfo):
283 pcurec = FindbadPCURecord.get_latest_by(plc_pcuid=get(fb_nodeinfo, 'plc_node_stats.pcu_ids')[0])
284 pcuinfo = pcurec.to_dict()
285 if verify(dict_query, pcuinfo):
286 nodenames.append(noderec.hostname)
287 str = "cmdhttps/locfg.pl -s %s -f iloxml/License.xml -u %s -p '%s' | grep MESSAGE" % \
288 (reboot.pcu_name(pcuinfo), pcuinfo['username'], pcuinfo['password'])
289 pcunames.append(pcuinfo['plc_pcuid'])
290 return (nodenames, pcunames)
292 def node_select(str_query, nodelist=None, fb=None):
295 if str_query is None: return hostnames
298 dict_query = query_to_dict(str_query)
301 for node in nodelist:
302 #if nodelist is not None:
303 # if node not in nodelist: continue
307 #fb_noderec = FindbadNodeRecord.query.filter(FindbadNodeRecord.hostname==node).order_by(FindbadNodeRecord.date_checked.desc()).first()
308 fb_noderec = FindbadNodeRecord.get_latest_by(hostname=node)
310 print traceback.print_exc()
314 fb_nodeinfo = fb_noderec.to_dict()
316 #fb_nodeinfo['pcu'] = color_pcu_state(fb_nodeinfo)
317 #if 'plcnode' in fb_nodeinfo:
318 # fb_nodeinfo.update(fb_nodeinfo['plcnode'])
320 #if verifyDBrecord(dict_query, fb_nodeinfo):
321 if verify(dict_query, fb_nodeinfo):
322 #print fb_nodeinfo.keys()
323 #print node #fb_nodeinfo
324 hostnames.append(node)
326 #print "NO MATCH", node
334 from monitor import parser as parsermodule
335 parser = parsermodule.getParser()
337 parser.set_defaults(node=None, fromtime=None, select=None, list=None, listkeys=False,
338 pcuselect=None, nodelist=None, daysdown=None, fields=None)
339 parser.add_option("", "--daysdown", dest="daysdown", action="store_true",
340 help="List the node state and days down...")
341 parser.add_option("", "--select", dest="select", metavar="key=value",
342 help="List all nodes with the given key=value pattern")
343 parser.add_option("", "--fields", dest="fields", metavar="key,list,...",
344 help="a list of keys to display for each entry.")
345 parser.add_option("", "--list", dest="list", action="store_true",
346 help="Write only the hostnames as output.")
347 parser.add_option("", "--pcuselect", dest="pcuselect", metavar="key=value",
348 help="List all nodes with the given key=value pattern")
349 parser.add_option("", "--nodelist", dest="nodelist", metavar="nodelist.txt",
350 help="A list of nodes to bring out of debug mode.")
351 parser.add_option("", "--listkeys", dest="listkeys", action="store_true",
352 help="A list of nodes to bring out of debug mode.")
353 parser.add_option("", "--fromtime", dest="fromtime", metavar="YYYY-MM-DD",
354 help="Specify a starting date from which to begin the query.")
356 parser = parsermodule.getParser(['defaults'], parser)
357 config = parsermodule.parse_args(parser)
361 archive = database.SPickle(path)
362 d = datetime_fromstr(config.fromtime)
363 glob_str = "%s*.production.findbad.pkl" % d.strftime("%Y-%m-%d")
366 file = glob.glob(glob_str)[0]
367 #print "loading %s" % file
369 fb = archive.load(file[:-4])
371 #fbnodes = FindbadNodeRecord.select(FindbadNodeRecord.q.hostname, orderBy='date_checked',distinct=True).reversed()
377 nodelist = util.file.getListFromFile(config.nodelist)
379 # NOTE: list of nodes should come from findbad db. Otherwise, we
380 # don't know for sure that there's a record in the db..
381 plcnodes = plccache.l_nodes
382 nodelist = [ node['hostname'] for node in plcnodes ]
383 #nodelist = ['planetlab-1.cs.princeton.edu']
386 if config.select is not None and config.pcuselect is not None:
387 nodelist = node_select(config.select, nodelist, fb)
388 nodelist, pculist = pcu_select(config.pcuselect, nodelist)
389 elif config.select is not None:
390 nodelist = node_select(config.select, nodelist, fb)
391 elif config.pcuselect is not None:
392 nodelist, pculist = pcu_select(config.pcuselect, nodelist)
398 for node in nodelist:
401 if node not in nodelist:
405 # Find the most recent record
406 fb_noderec = FindbadNodeRecord.query.filter(FindbadNodeRecord.hostname==node).order_by(FindbadNodeRecord.date_checked.desc()).first()
408 print traceback.print_exc()
412 fb_nodeinfo = fb_noderec.to_dict()
413 print "Primary keys available in the findbad object:"
414 for key in fb_nodeinfo.keys():
423 daysdown_print_nodeinfo(fb_nodeinfo, node)
425 fb_nodeinfo = fb_noderec.to_dict()
428 fields = config.fields.split(",")
432 fb_print_nodeinfo(fb_nodeinfo, node, fields)
433 elif not config.select and 'state' in fb_nodeinfo:
434 fb_print_nodeinfo(fb_nodeinfo, node)
438 if __name__ == "__main__":