4199e6ee9360afc4ac9a818142c1db4ae0fdf3a3
[infrastructure.git] / scripts / manage-infrastructure.py
1 #!/usr/bin/python
2
3 import os.path, sys
4 import re
5 import subprocess
6 from optparse import OptionParser
7
8 class Infrastructure:
9
10     # everything in the onelab.eu domain
11     domain = 'pl.sophia.inria.fr'
12     build_boxes = [ "devel", "liquid", "reed", "velvet", ]
13     plc_boxes = [ "testplc" ]
14     testmaster = 'testmaster'
15     testmaster_boxes = [ testmaster ]
16     # cache the list of qemu boxes in ~/.qemu-boxes
17     # this can be refreshed by running -c
18     qemu_boxes=[]
19
20     def cache_file (self): return os.path.expanduser("~/.qemu-boxes")
21
22     def load_cache (self):
23         cache=self.cache_file()
24         if os.path.isfile(cache):
25             self.qemu_boxes=file(cache).read().split()
26         self.test_boxes = self.plc_boxes + self.qemu_boxes
27
28     # run LocalTestResources on testmaster
29     def refresh_cache (self):
30         retrieved= \
31             self.backquote_ssh(self.fqdn(self.testmaster),['LocalTestResources.py'],trash_err=True)
32         remove="."+Infrastructure.domain
33         retrieved = [ x.replace(remove,"").strip() for x in retrieved.split()]
34         self.qemu_boxes = retrieved
35         cache=self.cache_file()
36         file(cache,'w').write(' '.join(self.qemu_boxes)+'\n')
37         print "New contents of %s:"%cache
38         print file(cache).read(),
39
40     def __init__ (self):
41         # dummy defaults
42         self.boxes = []
43         self.do_tracker_qemus = False
44         self.do_tracker_plcs = False
45         self.load_cache()
46
47     def fqdn (self, box):
48         return "%s.%s"%(box,self.domain)
49
50     ssh_command=['ssh','-o','ConnectTimeout=3']
51     @staticmethod
52     def root (box): return "root@%s"%box
53
54     @staticmethod
55     def ssh(box):
56         return Infrastructure.ssh_command + [ Infrastructure.root(box) ]
57
58     def header (self,message):
59         print "===============",message
60         sys.stdout.flush()
61
62     def run (self,argv,message, trash_err=False):
63         if self.options.dry_run:
64             print 'DRY_RUN:',
65             print " ".join(argv)
66             return 0
67         else:
68             if message: self.header(message)
69             if not trash_err:
70                 return subprocess.call(argv)
71             else:
72                 return subprocess.call(argv,stderr=file('/dev/null','w'))
73                 
74     def run_ssh (self, box, argv, message, trash_err=False):
75         result=self.run (self.ssh(box) + argv, message, trash_err)
76         if result!=0:
77             print "WARNING: failed to run %s on %s"%(" ".join(argv),box)
78         return result
79
80     def backquote (self, argv, trash_err=False):
81         if not trash_err:
82             return subprocess.Popen(argv,stdout=subprocess.PIPE).communicate()[0]
83         else:
84             return subprocess.Popen(argv,stdout=subprocess.PIPE,stderr=file('/dev/null','w')).communicate()[0]
85
86     def backquote_ssh (self, box, argv, trash_err=False):
87         # first probe the ssh link
88         hostname=self.backquote ( self.ssh(box) + [ "hostname"], trash_err=True )
89         if not hostname:
90             print "%s unreachable"%self.root(box)
91             return ''
92         else:
93             return self.backquote( ['ssh',self.root(box)] + argv, trash_err)
94
95     def reboot (self,box):
96         command=['ssh',self.root(box),'shutdown','-r','now']
97         self.run (command,"Rebooting %s"%box)
98
99     def handle_tracker_plcs (self):
100         box = self.fqdn (self.testmaster)
101         filename="tracker-plcs"
102         if not self.options.probe:
103             command=["rm","-rf",filename]
104             self.run_ssh(box,command,"Cleaning up %s on %s"%(filename,box))
105         else:
106             self.header ("++++++++++ Inspecting %s on %s"%(filename,box))
107             read_command = ["cat",filename]
108             trackers=self.backquote_ssh(box,read_command)
109             for tracker in trackers.split('\n'):
110                 if not tracker: continue
111                 try:
112                     tracker=tracker.strip()
113                     (hostname,buildname,plcname)=tracker.split('@')
114                     print self.margin_outline(plcname),tracker
115                 except:
116                     print self.margin(""),tracker
117
118     def handle_tracker_qemus (self):
119         box = self.fqdn (self.testmaster)
120         filename="tracker-qemus"
121         if not self.options.probe:
122             command=["rm","-rf",filename]
123             self.run_ssh(box,command,"Cleaning up %s on %s"%(filename,box))
124         else:
125             self.header ("++++++++++ Inspecting %s on %s"%(filename,box))
126             read_command = ["cat",filename]
127             trackers=self.backquote_ssh(box,read_command)
128             for tracker in trackers.split('\n'):
129                 if not tracker: continue
130                 try:
131                     tracker=tracker.strip()
132                     [hostname,buildname,nodename]=tracker.split('@')
133                     nodename=nodename.split('.')[0]
134                     print self.margin_outline(nodename),tracker
135                 except:
136                     print self.margin(""),tracker
137
138     def handle_build_box (self,box):
139         if not self.options.probe:
140             self.reboot(box)
141         else:
142             command=['uptime']
143             uptime=self.backquote_ssh(box,command,True).strip()
144
145             command=['pgrep','build']
146             if self.options.dry_run:
147                 self.run_ssh(box,command,None)
148             else:
149                 pids=self.backquote_ssh(box,command,True)
150                 if not pids:
151                     self.header ('No build process on %s (%s)'%(box,uptime))
152                 else:
153                     command=['ps','-o','pid,command'] + [ pid for pid in pids.split("\n") if pid]
154                     self.run_ssh(box,command,"Active build processes on %s (%s)"%(box,uptime),True)
155
156     # this one is more accurate as it locates processes in the vservers as well
157     # but it's so sloooowww
158     def handle_build_box_deep (self,box):
159         if not self.options.probe:
160             self.reboot(box)
161         else:
162             command=['uptime']
163             uptime=self.backquote_ssh(box,command,True).strip()
164
165             command=['vps','-e']
166             if self.options.dry_run:
167                 self.run_ssh(box,command,None)
168             else:
169                 # simulate grep vbuild
170                 vps_lines=[ line for line in self.backquote_ssh(box,command,True).split("\n")
171                             if line.find('vbuild') >= 0]
172                 pids=[ line.split()[0] for line in vps_lines ]
173                 if not pids:
174                     self.header ('No build process on %s (%s)'%(box,uptime))
175                 else:
176                     command=['vps','-o','pid,command'] + pids
177                     self.run_ssh(box,command,"Active build processes on %s (%s)"%(box,uptime),True)
178
179
180     vplc_matcher = re.compile(".*(vplc[0-9]+$)")
181     def vplcname (self, vservername):
182         match = self.vplc_matcher.match(vservername)
183         if match: return match.groups(0)
184         else: return ""
185
186     margin_format="%-14s"
187     def margin(self,string): return self.margin_format%string
188     def outline (self, string): return '== %s =='%string
189     def margin_outline (self, string): return self.margin(self.outline(string))
190
191     def handle_plc_box (self,box):
192 # initial approach was to first scan vserver-stat, but it's not needed
193         if not self.options.probe:
194 #            # remove mark for all running servers to avoid resurrection
195 #            if vserver_names:
196 #                bash="; ".join( [ "rm -f /etc/vservers/%s/apps/init/mark"%vs for vs in vserver_names ] )
197 #                stop_command=['bash','-c',"'" + bash + "'"]
198 #                self.run_ssh(box,stop_command,"Removing mark on running vservers on %s"%box)
199             # just trash all marks 
200             stop_command=['rm','-rf','/etc/vservers/*/apps/init/mark']
201             self.run_ssh(box,stop_command,"Removing all vserver marks on %s"%box)
202             if not self.options.soft:
203                 self.reboot(box)
204             else:
205                 self.run_ssh(box,['service','util-vserver','stop'],"Stopping all running vservers")
206             return
207         # even for rebooting we need to scan vserver-stat to stop the vservers properly
208         vserver_names=[]
209         command=['vserver-stat']
210         if self.options.dry_run:
211             self.run_ssh(box,command,"Active vservers on %s"%box)
212         # try to find fullname (vserver_stat truncates to a ridiculously short name)
213         self.header ("vserver map on %s"%box)
214         # fetch the contexts for all vservers on that box
215         map_command=['grep','.','/etc/vservers/*/context','/dev/null',]
216         context_map=self.backquote_ssh (box,map_command)
217         # at this point we have a set of lines like
218         # /etc/vservers/2010.01.20--k27-f12-32-vplc03/context:40144
219         ctx_dict={}
220         for map_line in context_map.split("\n"):
221             if not map_line: continue
222             [path,xid] = map_line.split(':')
223             ctx_dict[xid]=os.path.basename(os.path.dirname(path))
224         # at this point ctx_id maps context id to vservername
225
226         vserver_stat = self.backquote_ssh (box,command)
227         for vserver_line in vserver_stat.split("\n"):
228             if not vserver_line: continue
229             context=vserver_line.split()[0]
230             if context=="CTX": 
231                 print self.margin(""),vserver_line
232                 continue
233             longname=ctx_dict[context]
234             vserver_names.append(longname)
235             print self.margin_outline(self.vplcname(longname)),"%(vserver_line)s [=%(longname)s]"%locals()
236
237     vnode_matcher = re.compile(".*(vnode[0-9]+)")
238     def vnodename (self, ps_line):
239         match = self.vnode_matcher.match(ps_line)
240         if match: return match.groups(0)
241         else: return ""
242
243     def handle_qemu_box (self,box):
244         if not self.options.probe:
245             if not self.options.soft:
246                 self.reboot(box)
247             else:
248                 self.run_ssh(box,['pkill','qemu'],"Killing qemu instances")
249         else:
250             command=['lsmod']
251             modules=self.backquote_ssh(box,command).split('\n')
252             kqemu_msg='*NO kqemu/kmv_intel MODULE LOADED*'
253             for module in modules:
254                 if module.find('kqemu')==0:
255                     kqemu_msg='kqemu module loaded'
256                 # kvm might be loaded without vkm_intel (we dont have AMD)
257                 elif module.find('kvm_intel')==0:
258                     kqemu_msg='kvm_intel module loaded'
259             
260             command=['pgrep','qemu']
261             if self.options.dry_run:
262                 self.run_ssh(box,command,None)
263             else:
264                 pids=self.backquote_ssh(box,command)
265                 if not pids:
266                     self.header ('No qemu process on %s (%s)'%(box,kqemu_msg))
267                 else:
268                     self.header ("Active qemu processes on %s (%s)"%(box,kqemu_msg))
269                     command=['ps','-o','pid,command'] + [ pid for pid in pids.split("\n") if pid]
270                     ps_lines = self.backquote_ssh (box,command).split("\n")
271                     for ps_line in ps_lines:
272                         if not ps_line or ps_line.find('PID') >=0 : continue
273                         print self.margin_outline(self.vnodename(ps_line)), ps_line
274
275     # the ouput of ps -o pid,command gives us <pid> bash <buildname>/run_log
276     def testmaster_buildname (self, ps_line):
277         chunks=ps_line.split()
278         path=chunks[2]
279         [buildname,command]=path.split('/')
280         return buildname
281
282     def handle_testmaster_box (self, box):
283         if not self.options.probe: 
284             pass
285         else:
286             command=['pgrep','run_log']
287             if self.options.dry_run:
288                 self.run_ssh(box,command,None)
289             else:
290                 pids=self.backquote_ssh(box,command)
291                 if not pids:
292                     self.header ('No run_log process on %s'%box)
293                 else:
294                     self.header ("Active run_log processes on %s"%(box))
295                     command=['ps','-o','pid,command'] + [ pid for pid in pids.split("\n") if pid]
296                     ps_lines = self.backquote_ssh (box,command).split("\n")
297                     for ps_line in ps_lines:
298                         if not ps_line or ps_line.find('PID') >=0 : continue
299                         print self.margin_outline(self.testmaster_buildname(ps_line)), ps_line
300         
301
302     def handle_box(self,box,type):
303         if box in self.qemu_boxes:
304             if type=="qemu": self.handle_qemu_box(self.fqdn(box))
305         elif box in self.plc_boxes:
306             if type=="plc":  self.handle_plc_box(self.fqdn(box))
307         elif box in self.testmaster_boxes:
308             if type=='testmaster': self.handle_testmaster_box(self.fqdn(box))
309         elif type=="build":
310             if self.options.deep:
311                 self.handle_build_box_deep(self.fqdn(box))
312             else:
313                 self.handle_build_box(self.fqdn(box))
314
315     def handle_disk (self,box):
316         box=self.fqdn(box)
317         return self.run_ssh(box,["df","-h",],"Disk space on %s"%box)
318
319     def main (self):
320         usage="""%prog [options] [hostname..(s)]
321 Default is to act on test boxes only"""
322         parser = OptionParser (usage=usage)
323         parser.add_option ("-n","--dry-run",action="store_true",dest="dry_run",default=False,
324                            help="Dry run")
325         parser.add_option ("-r","--reboot", action="store_false",dest="probe",default=True,
326                            help="Actually reset/reboot stuff instead of just probing it")
327         parser.add_option ("-s","--soft",action="store_true",dest="soft",default=False,
328                            help="Soft reset instead of hard reboot of the boxes")
329         # no need for -p = probe, as this is the default
330         parser.add_option ("-p","--plc", action="store_true",dest="plc_only",default=False,
331                            help="Acts on the plc box only")
332
333         parser.add_option ("-e","--deep",action="store_true", dest="deep", default=False,
334                            help="on build boxes, shows vbuild processes in vservers as well; signif. slower")
335
336         parser.add_option ("-a","--all",action="store_true",dest="all_boxes",default=False,
337                            help="Acts on build and test boxes")
338         parser.add_option ("-b","--build",action="store_true",dest="build_only",default=False,
339                            help="Acts on build boxes only")
340         parser.add_option ("-q","--qemu",action="store_true",dest="qemu_only",default=False,
341                            help="Only acts on the qemu boxes")
342         parser.add_option ("-t","--trackers",action="store_true",dest="trackers_only",default=False,
343                            help="Only wipes trackers")
344         parser.add_option ("-m","--master",action="store_true",dest="testmaster_only",default=False,
345                            help="Display the testmaster status")
346         parser.add_option ("-d","--disk",action="store_true",dest="show_disk",default=False,
347                            help="Only inspects disk status")
348         parser.add_option ("-c","--refresh-cache",action="store_true",dest="refresh_cache", default=False,
349                            help="Refresh cached list of qemu boxes at testmaster - implies -q")
350
351         (self.options,args) = parser.parse_args()
352
353         # -c implies -q
354         if self.options.refresh_cache:
355             self.options.qemu_only=True
356             self.refresh_cache()
357
358         # use given hostnames if provided
359         if args:
360             self.boxes=args
361             # if hostnames are specified, let's stay on the safe side and don't reset trackers
362             self.do_tracker_plcs = False
363             self.do_tracker_qemus = False
364         elif self.options.all_boxes:
365             self.boxes=self.test_boxes + self.build_boxes + self.testmaster_boxes
366             self.do_tracker_plcs = True
367             self.do_tracker_qemus = True
368         elif self.options.build_only:
369             self.boxes=self.build_boxes
370             self.do_tracker_plcs = False
371             self.do_tracker_qemus = False
372         elif self.options.qemu_only:
373             self.boxes=self.qemu_boxes
374             self.do_tracker_plcs = False
375             self.do_tracker_qemus = True
376         elif self.options.plc_only:
377             self.boxes=self.plc_boxes
378             self.do_tracker_plcs = True
379             self.do_tracker_qemus = False
380         elif self.options.testmaster_only:
381             self.boxes=self.testmaster_boxes
382             self.do_tracker_plcs = False
383             self.do_tracker_qemus = False
384         elif self.options.trackers_only:
385             self.boxes = []
386             self.do_tracker_plcs = True
387             self.do_tracker_qemus = True
388         # default
389         else:
390             self.boxes = self.test_boxes
391             self.do_tracker_plcs = True
392             self.do_tracker_qemus = True
393
394         if self.options.show_disk:
395             for box in self.boxes: self.handle_disk(box)
396             return
397
398         # PLCS
399         if self.do_tracker_plcs:self.handle_tracker_plcs ()
400         for box in self.boxes:  self.handle_box (box,"plc")
401         # QEMU
402         if self.do_tracker_qemus:self.handle_tracker_qemus ()
403         for box in self.boxes:  self.handle_box (box,"qemu")
404         # ALL OTHERS
405         for box in self.boxes:  self.handle_box (box,"build")
406         # TESTMASTER
407         for box in self.boxes:  self.handle_box (box,"testmaster")
408
409 if __name__ == "__main__":
410     Infrastructure().main()