8 from config import config
9 from optparse import OptionParser
10 parser = OptionParser()
11 parser.set_defaults(filename="", increment=False, dbname="findbadnodes", cachenodes=False)
12 parser.add_option("-f", "--nodes", dest="filename", metavar="FILE",
13 help="Provide the input file for the node list")
14 parser.add_option("", "--cachenodes", action="store_true",
15 help="Cache node lookup from PLC")
16 parser.add_option("", "--dbname", dest="dbname", metavar="FILE",
17 help="Specify the name of the database to which the information is saved")
18 parser.add_option("-i", "--increment", action="store_true", dest="increment",
19 help="Increment round number to force refresh or retry")
20 config = config(parser)
24 COMON_COTOPURL= "http://summer.cs.princeton.edu/status/tabulator.cgi?" + \
25 "table=table_nodeview&" + \
26 "dumpcols='name,resptime,sshstatus,uptime,lastcotop'&" + \
29 #"select='lastcotop!=0'"
32 plc_lock = threading.Lock()
34 externalState = {'round': round, 'nodes': {}}
44 def collectPingAndSSH(nodename, cohash):
45 ### RUN PING ######################
47 (oval,eval) = ping.run_noexcept("ping -c 1 -q %s | grep rtt" % nodename)
53 values['ping'] = "NOPING"
55 values['ping'] = "PING"
57 ### RUN SSH ######################
59 ssh = soltesz.SSH('root', nodename)
62 (oval, eval) = ssh.run_noexcept('echo `uname -a ; ls /tmp/bm.log`')
64 if "2.6.17" in oval or "2.6.20" in oval:
67 values['category'] = 'ALPHA'
68 values['state'] = 'DEBUG'
70 values['category'] = 'ALPHA'
71 values['state'] = 'BOOT'
72 elif "2.6.12" in oval or "2.6.10" in oval:
74 values['category'] = 'PROD'
76 values['state'] = 'DEBUG'
78 values['state'] = 'BOOT'
80 b_getbootcd_id = False
82 values['category'] = 'OLDBOOTCD'
83 values['state'] = 'DEBUG'
86 values['category'] = 'UNKNOWN'
88 values['state'] = 'DEBUG'
90 values['state'] = 'BOOT'
93 b_getbootcd_id = False
94 values['ssh'] = 'NOSSH'
95 values['category'] = 'ERROR'
96 values['state'] = 'DOWN'
99 values['kernel'] = val
102 # try to get BootCD for all nodes that are not 2.4 nor inaccessible
103 (oval, eval) = ssh.run_noexcept('cat /mnt/cdrom/bootme/ID')
106 values['bootcd'] = val
108 values['category'] = 'OLDBOOTCD'
110 values['bootcd'] = ""
112 values['bootcd'] = ""
114 # TODO: get bm.log for debug nodes.
117 if nodename in cohash:
118 values['comonstats'] = cohash[nodename]
120 values['comonstats'] = {'resptime': '-1',
124 # include output value
125 ### GET PLC NODE ######################
130 d_node = plc.getNodes({'hostname': nodename}, ['pcu_ids', 'site_id', 'last_contact'])
134 traceback.print_exc()
137 if b_except: return (None, None)
140 if d_node and len(d_node) > 0:
141 pcu = d_node[0]['pcu_ids']
143 values['pcu'] = "PCU"
145 values['pcu'] = "NOPCU"
146 site_id = d_node[0]['site_id']
147 last_contact = d_node[0]['last_contact']
148 values['plcnode'] = {'status' : 'SUCCESS',
151 'last_contact': last_contact}
153 values['pcu'] = "UNKNOWN"
154 values['plcnode'] = {'status' : "GN_FAILED"}
157 ### GET PLC SITE ######################
162 d_site = plc.getSites({'site_id': site_id},
163 ['max_slices', 'slice_ids', 'node_ids', 'login_base'])
167 traceback.print_exc()
170 if b_except: return (None, None)
172 if d_site and len(d_site) > 0:
173 max_slices = d_site[0]['max_slices']
174 num_slices = len(d_site[0]['slice_ids'])
175 num_nodes = len(d_site[0]['node_ids'])
176 loginbase = d_site[0]['login_base']
177 values['plcsite'] = {'num_nodes' : num_nodes,
178 'max_slices' : max_slices,
179 'num_slices' : num_slices,
180 'login_base' : loginbase,
181 'status' : 'SUCCESS'}
183 values['plcsite'] = {'status' : "GS_FAILED"}
185 return (nodename, values)
187 def recordPingAndSSH(request, result):
190 (nodename, values) = result
192 if values is not None:
193 global_round = externalState['round']
194 externalState['nodes'][nodename]['values'] = values
195 externalState['nodes'][nodename]['round'] = global_round
198 print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values'])
199 soltesz.dbDump(config.dbname, externalState)
201 # this will be called when an exception occurs within a thread
202 def handle_exception(request, result):
203 print "Exception occured in request %s" % request.requestID
205 print "Result: %s" % i
208 def checkAndRecordState(l_nodes, cohash):
211 global_round = externalState['round']
213 tp = threadpool.ThreadPool(20)
215 # CREATE all the work requests
216 for nodename in l_nodes:
217 if nodename not in externalState['nodes']:
218 externalState['nodes'][nodename] = {'round': 0, 'values': []}
220 node_round = externalState['nodes'][nodename]['round']
221 if node_round < global_round:
222 # recreate node stats when refreshed
223 #print "%s" % nodename
224 req = threadpool.WorkRequest(collectPingAndSSH, [nodename, cohash], {},
225 None, recordPingAndSSH, handle_exception)
228 # We just skip it, since it's "up to date"
230 print "%d %s %s" % (count, nodename, externalState['nodes'][nodename]['values'])
233 # WAIT while all the work requests are processed.
238 except KeyboardInterrupt:
241 except threadpool.NoResultsPending:
242 print "All results collected."
250 externalState = soltesz.if_cached_else(1, config.dbname, lambda : externalState)
253 # update global round number to force refreshes across all nodes
254 externalState['round'] += 1
256 cotop = comon.Comon()
257 # lastcotop measures whether cotop is actually running. this is a better
258 # metric than sshstatus, or other values from CoMon
259 cotop_url = COMON_COTOPURL
261 # history information for all nodes
262 cohash = cotop.coget(cotop_url)
264 if config.filename == "":
265 l_nodes = syncplcdb.create_plcdb()
266 l_nodes = [node['hostname'] for node in l_nodes]
267 #l_nodes = cohash.keys()
269 l_nodes = config.getListFromFile(config.filename)
271 checkAndRecordState(l_nodes, cohash)
276 if __name__ == '__main__':
279 except Exception, err:
280 print "Exception: %s" % err
281 print "Saving data... exitting."
282 soltesz.dbDump(config.dbname, externalState)