5 from monitor import database
6 from nodecommon import *
7 from unified_model import Record
16 from monitor.pcu import reboot
17 from monitor.wrapper import plc, plccache
18 api = plc.getAuthAPI()
20 from monitor.database.infovacuum import FindbadNodeRecordSync, FindbadNodeRecord
21 from monitor.database.dborm import mon_session as session
22 from monitor import util
23 from monitor import config
26 class NoKeyException(Exception): pass
28 def daysdown_print_nodeinfo(fbnode, hostname):
29 fbnode['hostname'] = hostname
30 fbnode['daysdown'] = Record.getStrDaysDown(fbnode)
31 fbnode['intdaysdown'] = Record.getDaysDown(fbnode)
33 print "%(intdaysdown)5s %(hostname)-44s | %(state)10.10s | %(daysdown)s" % fbnode
35 def fb_print_nodeinfo(fbnode, hostname, fields=None):
36 #fbnode['hostname'] = hostname
37 #fbnode['checked'] = diff_time(fbnode['checked'])
38 if fbnode['bootcd_version']:
39 fbnode['bootcd_version'] = fbnode['bootcd_version'].split()[-1]
41 fbnode['bootcd_version'] = "unknown"
42 fbnode['pcu'] = color_pcu_state(fbnode)
45 if ( fbnode['observed_status'] is not None and \
46 'DOWN' in fbnode['observed_status'] ) or \
47 fbnode['kernel_version'] is None:
48 fbnode['kernel_version'] = ""
50 fbnode['kernel_version'] = fbnode['kernel_version'].split()[2]
52 if fbnode['plc_node_stats'] is not None:
53 fbnode['boot_state'] = fbnode['plc_node_stats']['boot_state']
55 fbnode['boot_state'] = "unknown"
58 if len(fbnode['nodegroups']) > 0:
59 fbnode['category'] = fbnode['nodegroups'][0]
61 #print "ERROR!!!!!!!!!!!!!!!!!!!!!"
64 print "%(hostname)-45s | %(date_checked)11.11s | %(boot_state)5.5s| %(observed_status)8.8s | %(ssh_status)5.5s | %(pcu)6.6s | %(bootcd_version)6.6s | %(kernel_version)s" % fbnode
68 format += "%%(%s)s " % f
72 indexes = path.split(".")
76 indexes = path.split(".")
80 values = values[index]
82 raise NoKeyException(index)
85 def verifyType(constraints, data):
87 constraints is a list of key, value pairs.
88 # [ {... : ...}==AND , ... , ... , ] == OR
91 for con in constraints:
92 #print "con: %s" % con
93 if len(con.keys()) == 0:
98 for key in con.keys():
99 #print "looking at key: %s" % key
107 if o.name() == "Match":
108 if get(data,key) is not None:
109 value_re = re.compile(o.value)
110 con_and_true = con_and_true & (value_re.search(get(data,key)) is not None)
113 elif o.name() == "ListMatch":
114 if get(data,key) is not None:
116 for listitem in get(data,key):
117 value_re = re.compile(o.value)
118 if value_re.search(listitem) is not None:
121 con_and_true = con_and_true & match
124 elif o.name() == "Is":
125 con_and_true = con_and_true & (get(data,key) == o.value)
126 elif o.name() == "FilledIn":
127 con_and_true = con_and_true & (len(get(data,key)) > 0)
128 elif o.name() == "PortOpen":
129 if get(data,key) is not None:
131 con_and_true = con_and_true & (v[str(o.value)] == "open")
135 value_re = re.compile(o.value)
136 con_and_true = con_and_true & (value_re.search(get(data,key)) is not None)
138 except NoKeyException, key:
139 print "missing key %s" % key,
141 #print "missing key %s" % key
142 #con_and_true = False
144 con_or_true = con_or_true | con_and_true
148 def verifyDBrecord(constraints, record):
150 constraints is a list of key, value pairs.
151 # [ {... : ...}==AND , ... , ... , ] == OR
153 def has_key(obj, key):
155 x = obj.__getattribute__(key)
160 def get_val(obj, key):
162 return obj.__getattribute__(key)
167 indexes = path.split("/")
168 value = get_val(obj,indexes[0])
169 if value is not None and len(indexes) > 1:
170 for key in indexes[1:]:
174 raise NoKeyException(key)
177 #print constraints, record
180 for con in constraints:
181 #print "con: %s" % con
182 if len(con.keys()) == 0:
187 for key in con.keys():
188 #print "looking at key: %s" % key
189 if has_key(record, key):
190 value_re = re.compile(con[key])
191 if type([]) == type(get(record,key)):
192 local_or_true = False
193 for val in get(record,key):
194 local_or_true = local_or_true | (value_re.search(val) is not None)
195 con_and_true = con_and_true & local_or_true
197 if get(record,key) is not None:
198 con_and_true = con_and_true & (value_re.search(get(record,key)) is not None)
200 print "missing key %s" % key,
203 con_or_true = con_or_true | con_and_true
207 def verify(constraints, data):
209 constraints is a list of key, value pairs.
210 # [ {... : ...}==AND , ... , ... , ] == OR
213 for con in constraints:
214 #print "con: %s" % con
215 if len(con.keys()) == 0:
220 for key in con.keys():
221 #print "looking at key: %s" % key
222 if first(key) in data:
223 value_re = re.compile(con[key])
224 if type([]) == type(get(data,key)):
225 local_or_true = False
226 for val in get(data,key):
227 local_or_true = local_or_true | (value_re.search(val) is not None)
228 con_and_true = con_and_true & local_or_true
230 if get(data,key) is not None:
231 con_and_true = con_and_true & (value_re.search(get(data,key)) is not None)
232 elif first(key) not in data:
233 print "missing key %s" % first(key)
235 con_or_true = con_or_true | con_and_true
239 def query_to_dict(query):
243 or_queries = query.split('||')
244 for or_query in or_queries:
245 and_queries = or_query.split('&&')
249 for and_query in and_queries:
250 (key, value) = and_query.split('=')
258 if 'plcnode' in fbdata:
259 if 'pcu_ids' in fbdata['plcnode']:
260 if len(fbdata['plcnode']['pcu_ids']) > 0:
264 def pcu_select(str_query, nodelist=None):
267 if str_query is None: return (nodenames, pcunames)
270 fbquery = FindbadNodeRecord.get_all_latest()
271 fb_nodelist = [ n.hostname for n in fbquery ]
273 fbpcuquery = FindbadPCURecord.get_all_latest()
274 fbpcu_list = [ p.plc_pcuid for p in fbpcuquery ]
276 dict_query = query_to_dict(str_query)
278 for noderec in fbquery:
279 if nodelist is not None:
280 if noderec.hostname not in nodelist: continue
282 fb_nodeinfo = noderec.to_dict()
283 if pcu_in(fb_nodeinfo):
284 pcurec = FindbadPCURecord.get_latest_by(plc_pcuid=get(fb_nodeinfo, 'plc_node_stats.pcu_ids')[0])
285 pcuinfo = pcurec.to_dict()
286 if verify(dict_query, pcuinfo):
287 nodenames.append(noderec.hostname)
288 str = "cmdhttps/locfg.pl -s %s -f iloxml/License.xml -u %s -p '%s' | grep MESSAGE" % \
289 (reboot.pcu_name(pcuinfo), pcuinfo['username'], pcuinfo['password'])
290 pcunames.append(pcuinfo['plc_pcuid'])
291 return (nodenames, pcunames)
293 def node_select(str_query, nodelist=None, fb=None):
296 if str_query is None: return hostnames
299 dict_query = query_to_dict(str_query)
302 for node in nodelist:
303 #if nodelist is not None:
304 # if node not in nodelist: continue
308 #fb_noderec = FindbadNodeRecord.query.filter(FindbadNodeRecord.hostname==node).order_by(FindbadNodeRecord.date_checked.desc()).first()
309 fb_noderec = FindbadNodeRecord.get_latest_by(hostname=node)
311 print traceback.print_exc()
315 fb_nodeinfo = fb_noderec.to_dict()
317 #fb_nodeinfo['pcu'] = color_pcu_state(fb_nodeinfo)
318 #if 'plcnode' in fb_nodeinfo:
319 # fb_nodeinfo.update(fb_nodeinfo['plcnode'])
321 #if verifyDBrecord(dict_query, fb_nodeinfo):
322 if verify(dict_query, fb_nodeinfo):
323 #print fb_nodeinfo.keys()
324 #print node #fb_nodeinfo
325 hostnames.append(node)
327 #print "NO MATCH", node
335 from monitor import parser as parsermodule
336 parser = parsermodule.getParser()
338 parser.set_defaults(node=None, fromtime=None, select=None, list=None, listkeys=False,
339 pcuselect=None, nodelist=None, daysdown=None, fields=None)
340 parser.add_option("", "--daysdown", dest="daysdown", action="store_true",
341 help="List the node state and days down...")
342 parser.add_option("", "--select", dest="select", metavar="key=value",
343 help="List all nodes with the given key=value pattern")
344 parser.add_option("", "--fields", dest="fields", metavar="key,list,...",
345 help="a list of keys to display for each entry.")
346 parser.add_option("", "--list", dest="list", action="store_true",
347 help="Write only the hostnames as output.")
348 parser.add_option("", "--pcuselect", dest="pcuselect", metavar="key=value",
349 help="List all nodes with the given key=value pattern")
350 parser.add_option("", "--nodelist", dest="nodelist", metavar="nodelist.txt",
351 help="A list of nodes to bring out of debug mode.")
352 parser.add_option("", "--listkeys", dest="listkeys", action="store_true",
353 help="A list of nodes to bring out of debug mode.")
354 parser.add_option("", "--fromtime", dest="fromtime", metavar="YYYY-MM-DD",
355 help="Specify a starting date from which to begin the query.")
357 parser = parsermodule.getParser(['defaults'], parser)
358 config = parsermodule.parse_args(parser)
362 archive = database.SPickle(path)
363 d = datetime_fromstr(config.fromtime)
364 glob_str = "%s*.production.findbad.pkl" % d.strftime("%Y-%m-%d")
367 file = glob.glob(glob_str)[0]
368 #print "loading %s" % file
370 fb = archive.load(file[:-4])
372 #fbnodes = FindbadNodeRecord.select(FindbadNodeRecord.q.hostname, orderBy='date_checked',distinct=True).reversed()
378 nodelist = util.file.getListFromFile(config.nodelist)
380 # NOTE: list of nodes should come from findbad db. Otherwise, we
381 # don't know for sure that there's a record in the db..
382 plcnodes = plccache.l_nodes
383 nodelist = [ node['hostname'] for node in plcnodes ]
384 #nodelist = ['planetlab-1.cs.princeton.edu']
387 if config.select is not None and config.pcuselect is not None:
388 nodelist = node_select(config.select, nodelist, fb)
389 nodelist, pculist = pcu_select(config.pcuselect, nodelist)
390 elif config.select is not None:
391 nodelist = node_select(config.select, nodelist, fb)
392 elif config.pcuselect is not None:
393 nodelist, pculist = pcu_select(config.pcuselect, nodelist)
399 for node in nodelist:
402 if node not in nodelist:
406 # Find the most recent record
407 fb_noderec = FindbadNodeRecord.query.filter(FindbadNodeRecord.hostname==node).order_by(FindbadNodeRecord.date_checked.desc()).first()
409 print traceback.print_exc()
413 fb_nodeinfo = fb_noderec.to_dict()
414 print "Primary keys available in the findbad object:"
415 for key in fb_nodeinfo.keys():
424 daysdown_print_nodeinfo(fb_nodeinfo, node)
426 fb_nodeinfo = fb_noderec.to_dict()
429 fields = config.fields.split(",")
433 fb_print_nodeinfo(fb_nodeinfo, node, fields)
434 elif not config.select and 'state' in fb_nodeinfo:
435 fb_print_nodeinfo(fb_nodeinfo, node)
439 if __name__ == "__main__":