6 from optparse import OptionParser
10 # everything in the onelab.eu domain
11 domain = 'pl.sophia.inria.fr'
12 testmaster = 'testmaster'
13 build_boxes = [ "mirror", "liquid", "reed", "velvet", ]
14 plc_boxes = [ "testplc" ]
16 [ "qemu64-%d"%i for i in range (1,4) ] + \
17 [ "qemu32-%d"%i for i in range (1,6) ]
18 test_boxes = plc_boxes + qemu_boxes
23 self.do_tracker_qemus = False
24 self.do_tracker_plcs = False
27 return "%s.%s"%(box,self.domain)
29 ssh_command=['ssh','-o','ConnectTimeout=3']
31 def root (box): return "root@%s"%box
35 return BuildBoxes.ssh_command + [ BuildBoxes.root(box) ]
37 def header (self,message):
38 print "===============",message
40 def run (self,argv,message, trash_err=False):
41 if self.options.dry_run:
46 if message: self.header(message)
48 return subprocess.call(argv)
50 return subprocess.call(argv,stderr=file('/dev/null','w'))
52 def run_ssh (self, box, argv, message, trash_err=False):
53 result=self.run (self.ssh(box) + argv, message, trash_err)
55 print "WARNING: failed to run %s on %s"%(" ".join(argv),box)
58 def backquote (self, argv, trash_err=False):
60 return subprocess.Popen(argv,stdout=subprocess.PIPE).communicate()[0]
62 return subprocess.Popen(argv,stdout=subprocess.PIPE,stderr=file('/dev/null','w')).communicate()[0]
64 def backquote_ssh (self, box, argv, trash_err=False):
65 # print 'BACKQUOTE_SSH [%s] %s'%(box,' '.join(argv))
66 # first probe the ssh link
67 hostname=self.backquote ( self.ssh(box) + [ "hostname"], trash_err=True )
69 print "%s unreachable"%self.root(box)
72 return self.backquote( ['ssh',self.root(box)] + argv, trash_err)
74 def reboot (self,box):
75 command=['ssh',self.root(box),'shutdown','-r','now']
76 self.run (command,"Rebooting %s"%box)
78 def handle_tracker_plcs (self):
79 box = self.fqdn (self.testmaster)
80 filename="tracker-plcs"
81 if not self.options.probe:
82 command=["rm","-rf",filename]
83 self.run_ssh(box,command,"Cleaning up %s on %s"%(filename,box))
85 self.header ("++++++++++ Inspecting %s on %s"%(filename,box))
86 read_command = ["cat",filename]
87 trackers=self.backquote_ssh(box,read_command)
88 for tracker in trackers.split('\n'):
89 if not tracker: continue
91 tracker=tracker.strip()
92 [hostname,buildname]=tracker.split('@')
93 [left,plcname]=buildname.rsplit('-',1)
94 print self.margin_outline(plcname),tracker
96 print self.margin(""),tracker
98 def handle_tracker_qemus (self):
99 box = self.fqdn (self.testmaster)
100 filename="tracker-qemus"
101 if not self.options.probe:
102 command=["rm","-rf",filename]
103 self.run_ssh(box,command,"Cleaning up %s on %s"%(filename,box))
105 self.header ("++++++++++ Inspecting %s on %s"%(filename,box))
106 read_command = ["cat",filename]
107 trackers=self.backquote_ssh(box,read_command)
108 for tracker in trackers.split('\n'):
109 if not tracker: continue
111 tracker=tracker.strip()
112 [hostname,buildname,nodename]=tracker.split('@')
113 nodename=nodename.split('.')[0]
114 print self.margin_outline(nodename),tracker
116 print self.margin(""),tracker
118 def handle_build_box (self,box):
119 if not self.options.probe:
123 uptime=self.backquote_ssh(box,command,True).strip()
125 command=['pgrep','build']
126 if self.options.dry_run:
127 self.run_ssh(box,command,None)
129 pids=self.backquote_ssh(box,command,True)
131 self.header ('No build process on %s (%s)'%(box,uptime))
133 command=['ps','-o','pid,command'] + [ pid for pid in pids.split("\n") if pid]
134 self.run_ssh(box,command,"Active build processes on %s (%s)"%(box,uptime),True)
136 vplc_matcher = re.compile(".*(vplc[0-9]+$)")
137 def vplcname (self, vservername):
138 match = self.vplc_matcher.match(vservername)
139 if match: return match.groups(0)
142 margin_format="%-14s"
143 def margin(self,string): return self.margin_format%string
144 def outline (self, string): return '== %s =='%string
145 def margin_outline (self, string): return self.margin(self.outline(string))
147 def handle_plc_box (self,box):
148 if not self.options.probe:
151 command=['vserver-stat']
152 if self.options.dry_run:
153 self.run_ssh(box,command,"Active vservers on %s"%box)
155 # try to find fullname (vserver_stat truncates to a ridiculously short name)
157 self.header ("vserver map on %s"%box)
158 # fetch the contexts for all vservers on that box
159 map_command=['grep','.','/etc/vservers/*/context','/dev/null',]
160 context_map=self.backquote_ssh (box,map_command)
161 # at this point we have a set of lines like
162 # /etc/vservers/2010.01.20--k27-f12-32-vplc03/context:40144
164 for map_line in context_map.split("\n"):
165 if not map_line: continue
166 [path,xid] = map_line.split(':')
167 ctx_dict[xid]=os.path.basename(os.path.dirname(path))
168 # at this point ctx_id maps context id to vservername
170 vserver_stat = self.backquote_ssh (box,command)
171 for vserver_line in vserver_stat.split("\n"):
172 if not vserver_line: continue
173 context=vserver_line.split()[0]
175 print self.margin(""),vserver_line
177 longname=ctx_dict[context]
178 print self.margin_outline(self.vplcname(longname)),"%(vserver_line)s [=%(longname)s]"%locals()
180 self.run_ssh(box,command,"Fine-grained method failed - fallback to plain vserver-stat")
182 vnode_matcher = re.compile(".*(vnode[0-9]+)")
183 def vnodename (self, ps_line):
184 match = self.vnode_matcher.match(ps_line)
185 if match: return match.groups(0)
189 def handle_qemu_box (self,box):
190 if not self.options.probe:
194 modules=self.backquote_ssh(box,command).split('\n')
195 kqemu_msg='*NO kqemu MODULE LOADED*'
196 for module in modules:
197 if module.find('kqemu')==0:
200 command=['pgrep','qemu']
201 if self.options.dry_run:
202 self.run_ssh(box,command,None)
204 pids=self.backquote_ssh(box,command)
206 self.header ('No qemu process on %s'%box)
208 self.header ("Active qemu processes on %s (%s)"%(box,kqemu_msg))
209 command=['ps','-o','pid,command'] + [ pid for pid in pids.split("\n") if pid]
210 ps_lines = self.backquote_ssh (box,command).split("\n")
211 for ps_line in ps_lines:
212 if not ps_line or ps_line.find('PID') >=0 : continue
213 print self.margin_outline(self.vnodename(ps_line)), ps_line
215 def handle_box(self,box,type):
216 if box in self.qemu_boxes:
217 if type=="qemu": self.handle_qemu_box(self.fqdn(box))
218 elif box in self.plc_boxes:
219 if type=="plc": self.handle_plc_box(self.fqdn(box))
221 self.handle_build_box(self.fqdn(box))
224 usage="""%prog [options] [hostname..(s)]
225 Default is to act on test boxes only (with trackers clean)"""
226 parser = OptionParser (usage=usage)
227 parser.add_option ("-n","--dry-run",action="store_true",dest="dry_run",default=False,
229 parser.add_option ("-r","--reboot", action="store_false",dest="probe",default=True,
230 help="Actually reset/reboot stuff instead of just probing it")
231 # no need for -p = probe, as this is the default
232 parser.add_option ("-p","--plc", action="store_true",dest="plc_only",default=False,
233 help="Acts on the plc box only")
235 parser.add_option ("-a","--all",action="store_true",dest="all_boxes",default=False,
236 help="Acts on build and test boxes")
237 parser.add_option ("-b","--build",action="store_true",dest="build_only",default=False,
238 help="Acts on build boxes only")
239 parser.add_option ("-q","--qemu",action="store_true",dest="qemu_only",default=False,
240 help="Only acts on the qemu boxes")
241 parser.add_option ("-t","--trackers",action="store_true",dest="trackers_only",default=False,
242 help="Only wipes trackers")
244 (self.options,args) = parser.parse_args()
246 # use given hostnames if provided
249 # if hostnames are specified, let's stay on the safe side and don't reset trackers
250 self.do_tracker_plcs = False
251 self.do_tracker_qemus = False
252 elif self.options.all_boxes:
253 self.boxes=self.test_boxes + self.build_boxes
254 self.do_tracker_plcs = True
255 self.do_tracker_qemus = True
256 elif self.options.build_only:
257 self.boxes=self.build_boxes
258 self.do_tracker_plcs = False
259 self.do_tracker_qemus = False
260 elif self.options.qemu_only:
261 self.boxes=self.qemu_boxes
262 self.do_tracker_plcs = False
263 self.do_tracker_qemus = True
264 elif self.options.plc_only:
265 self.boxes=self.plc_boxes
266 self.do_tracker_plcs = True
267 self.do_tracker_qemus = False
268 elif self.options.trackers_only:
270 self.do_tracker_plcs = True
271 self.do_tracker_qemus = True
274 self.boxes = self.test_boxes
275 self.do_tracker_plcs = True
276 self.do_tracker_qemus = True
279 for box in self.boxes: self.handle_box (box,"build")
281 if self.do_tracker_plcs:self.handle_tracker_plcs ()
282 for box in self.boxes: self.handle_box (box,"plc")
284 if self.do_tracker_qemus:self.handle_tracker_qemus ()
285 for box in self.boxes: self.handle_box (box,"qemu")
287 if __name__ == "__main__":