system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16 from TestSlice import TestSlice
  17 from TestSliver import TestSliver
  18 from TestBox import TestBox
  19 from TestSsh import TestSsh
  20
  21 # step methods must take (self, options) and return a boolean
  22
  23 def standby(minutes):
  24         utils.header('Entering StandBy for %d mn'%minutes)
  25         time.sleep(60*minutes)
  26         return True
  27
  28 def standby_generic (func):
  29     def actual(self,options):
  30         minutes=int(func.__name__.split("_")[1])
  31         return standby(minutes)
  32     return actual
  33
  34 class TestPlc:
  35
  36     def __init__ (self,plc_spec):
  37         self.plc_spec=plc_spec
  38         self.path=os.path.dirname(sys.argv[0])
  39         self.test_ssh=TestSsh(self.plc_spec['hostname'],self.path)
  40         try:
  41             self.vserverip=plc_spec['vserverip']
  42             self.vservername=plc_spec['vservername']
  43             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  44             self.vserver=True
  45         except:
  46             self.vserver=False
  47             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  48         utils.header('Using API url %s'%self.url)
  49         self.server=xmlrpclib.Server(self.url,allow_none=True)
  50
  51     def name(self):
  52         name=self.plc_spec['name']
  53         if self.vserver:
  54             return name+"[%s]"%self.vservername
  55         else:
  56             return name+"[chroot]"
  57
  58     def hostname(self):
  59         return self.plc_spec['hostname']
  60
  61     def is_local (self):
  62         return self.test_ssh.is_local()
  63
  64     # define the API methods on this object through xmlrpc
  65     # would help, but not strictly necessary
  66     def connect (self):
  67         pass
  68
  69     def full_command(self,command):
  70         return self.test_ssh.to_host(self.host_to_guest(command))
  71
  72     def run_in_guest (self,command):
  73         return utils.system(self.full_command(command))
  74
  75     def run_in_host (self,command):
  76         return utils.system(self.test_ssh.to_host(command))
  77
  78     #command gets run in the chroot/vserver
  79     def host_to_guest(self,command):
  80         if self.vserver:
  81             return "vserver %s exec %s"%(self.vservername,command)
  82         else:
  83             return "chroot /plc/root %s"%TestSsh.backslash_shell_specials(command)
  84
  85     # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
  86     def copy_in_guest (self, localfile, remotefile, in_data=False):
  87         if in_data:
  88             chroot_dest="/plc/data"
  89         else:
  90             chroot_dest="/plc/root"
  91         if self.is_local():
  92             if not self.vserver:
  93                 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
  94             else:
  95                 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
  96         else:
  97             if not self.vserver:
  98                 utils.system("scp %s %s:%s/%s"%(localfile,self.hostname(),chroot_dest,remotefile))
  99             else:
 100                 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.hostname(),self.vservername,remotefile))
 101
 102
 103         # xxx quick n dirty
 104     def run_in_guest_piped (self,local,remote):
 105         return utils.system(local+" | "+self.full_command(remote))
 106
 107
 108     def auth_root (self):
 109         return {'Username':self.plc_spec['PLC_ROOT_USER'],
 110                 'AuthMethod':'password',
 111                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
 112                 'Role' : self.plc_spec['role']
 113                 }
 114     def locate_site (self,sitename):
 115         for site in self.plc_spec['sites']:
 116             if site['site_fields']['name'] == sitename:
 117                 return site
 118             if site['site_fields']['login_base'] == sitename:
 119                 return site
 120         raise Exception,"Cannot locate site %s"%sitename
 121
 122     def locate_node (self,nodename):
 123         for site in self.plc_spec['sites']:
 124             for node in site['nodes']:
 125                 if node['node_fields']['hostname'] == nodename:
 126                     return (site,node)
 127         raise Exception,"Cannot locate node %s"%nodename
 128
 129     def locate_key (self,keyname):
 130         for key in self.plc_spec['keys']:
 131             if key['name'] == keyname:
 132                 return key
 133         raise Exception,"Cannot locate key %s"%keyname
 134
 135     # all different hostboxes used in this plc
 136     def gather_hostBoxes(self):
 137         # maps on sites and nodes, return [ (host_box,test_node) ]
 138         tuples=[]
 139         for site_spec in self.plc_spec['sites']:
 140             test_site = TestSite (self,site_spec)
 141             for node_spec in site_spec['nodes']:
 142                 test_node = TestNode (self, test_site, node_spec)
 143                 if not test_node.is_real():
 144                     tuples.append( (test_node.host_box(),test_node) )
 145         # transform into a dict { 'host_box' -> [ hostnames .. ] }
 146         result = {}
 147         for (box,node) in tuples:
 148             if not result.has_key(box):
 149                 result[box]=[node]
 150             else:
 151                 result[box].append(node)
 152         return result
 153
 154     # a step for checking this stuff
 155     def showboxes (self,options):
 156         print 'showboxes'
 157         for (box,nodes) in self.gather_hostBoxes().iteritems():
 158             print box,":"," + ".join( [ node.name() for node in nodes ] )
 159         return True
 160
 161     # make this a valid step
 162     def kill_all_qemus(self,options):
 163         for (box,nodes) in self.gather_hostBoxes().iteritems():
 164             # this is the brute force version, kill all qemus on that host box
 165             TestBox(box,options.buildname).kill_all_qemus()
 166         return True
 167
 168     # make this a valid step
 169     def list_all_qemus(self,options):
 170         for (box,nodes) in self.gather_hostBoxes().iteritems():
 171             # push the script
 172             TestBox(box,options.buildname).copy("qemu_kill.sh")
 173             # this is the brute force version, kill all qemus on that host box
 174             TestBox(box,options.buildname).run_in_buildname("qemu_kill.sh -l")
 175         return True
 176
 177     # kill only the right qemus
 178     def force_kill_qemus(self,options):
 179         for (box,nodes) in self.gather_hostBoxes().iteritems():
 180             # push the script
 181             TestBox(box,options.buildname).copy("qemu_kill.sh")
 182             # the fine-grain version
 183             for node in nodes:
 184                 node.kill_qemu()
 185         return True
 186
 187     def clear_ssh_config (self,options):
 188         # install local ssh_config file as root's .ssh/config - ssh should be quiet
 189         # dir might need creation first
 190         self.run_in_guest("mkdir /root/.ssh")
 191         self.run_in_guest("chmod 700 /root/.ssh")
 192         # this does not work - > redirection somehow makes it until an argument to cat
 193         #self.run_in_guest_piped("cat ssh_config","cat > /root/.ssh/config")
 194         self.copy_in_guest("ssh_config","/root/.ssh/config",True)
 195         return True
 196
 197     #################### step methods
 198
 199     ### uninstall
 200     def uninstall_chroot(self,options):
 201         self.run_in_host('service plc safestop')
 202         #####detecting the last myplc version installed and remove it
 203         self.run_in_host('rpm -e myplc')
 204         ##### Clean up the /plc directory
 205         self.run_in_host('rm -rf  /plc/data')
 206         ##### stop any running vservers
 207         self.run_in_host('for vserver in $(ls /vservers/* | sed -e s,/vservers/,,) ; do vserver $vserver stop ; done')
 208         return True
 209
 210     def uninstall_vserver(self,options):
 211         self.run_in_host("vserver --silent %s delete"%self.vservername)
 212         return True
 213
 214     def uninstall(self,options):
 215         # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
 216         # it sounds safer to have the former uninstalled too
 217         # now the vserver method cannot be invoked for chroot instances as vservername is required
 218         if self.vserver:
 219             self.uninstall_vserver(options)
 220             self.uninstall_chroot(options)
 221         else:
 222             self.uninstall_chroot(options)
 223         return True
 224
 225     ### install
 226     def install_chroot(self,options):
 227         # nothing to do
 228         return True
 229
 230     def install_vserver(self,options):
 231         # we need build dir for vtest-init-vserver
 232         if self.is_local():
 233             # a full path for the local calls
 234             build_dir=self.path+"/build"
 235         else:
 236             # use a standard name - will be relative to HOME
 237             build_dir="options.buildname"
 238         # run checkout in any case - would do an update if already exists
 239         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 240         if self.run_in_host(build_checkout) != 0:
 241             raise Exception,"Cannot checkout build dir"
 242         # the repo url is taken from myplc-url
 243         # with the last two steps (i386/myplc...) removed
 244         repo_url = options.myplc_url
 245         for level in [ 'rpmname','arch' ]:
 246             repo_url = os.path.dirname(repo_url)
 247         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 248             (build_dir,self.vservername,repo_url,self.vserverip)
 249         if self.run_in_host(create_vserver) != 0:
 250             raise Exception,"Could not create vserver for %s"%self.vservername
 251         return True
 252
 253     def install(self,options):
 254         if self.vserver:
 255             return self.install_vserver(options)
 256         else:
 257             return self.install_chroot(options)
 258
 259     ### install_rpm
 260     def cache_rpm(self,url):
 261         self.run_in_host('rm -rf *.rpm')
 262         utils.header('Curling rpm from %s'%url)
 263         id= self.run_in_host('curl -O '+url)
 264         if (id != 0):
 265                 raise Exception,"Could not get rpm from  %s"%url
 266                 return False
 267         return True
 268
 269     def install_rpm_chroot(self,options):
 270         rpm = os.path.basename(options.myplc_url)
 271         if (not os.path.isfile(rpm)):
 272                 self.cache_rpm(options.myplc_url)
 273         utils.header('Installing the :  %s'%rpm)
 274         self.run_in_host('rpm -Uvh '+rpm)
 275         self.run_in_host('service plc mount')
 276         return True
 277
 278     def install_rpm_vserver(self,options):
 279         self.run_in_guest("yum -y install myplc-native")
 280         return True
 281
 282     def install_rpm(self,options):
 283         if self.vserver:
 284             return self.install_rpm_vserver(options)
 285         else:
 286             return self.install_rpm_chroot(options)
 287
 288     ###
 289     def configure(self,options):
 290         tmpname='%s.plc-config-tty'%(self.name())
 291         fileconf=open(tmpname,'w')
 292         for var in [ 'PLC_NAME',
 293                      'PLC_ROOT_PASSWORD',
 294                      'PLC_ROOT_USER',
 295                      'PLC_MAIL_ENABLED',
 296                      'PLC_MAIL_SUPPORT_ADDRESS',
 297                      'PLC_DB_HOST',
 298                      'PLC_API_HOST',
 299                      'PLC_WWW_HOST',
 300                      'PLC_BOOT_HOST',
 301                      'PLC_NET_DNS1',
 302                      'PLC_NET_DNS2']:
 303             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 304         fileconf.write('w\n')
 305         fileconf.write('q\n')
 306         fileconf.close()
 307         utils.system('cat %s'%tmpname)
 308         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 309         utils.system('rm %s'%tmpname)
 310         return True
 311
 312     # the chroot install is slightly different to this respect
 313     def start(self, options):
 314         if self.vserver:
 315             self.run_in_guest('service plc start')
 316         else:
 317             self.run_in_host('service plc start')
 318         return True
 319
 320     def stop(self, options):
 321         if self.vserver:
 322             self.run_in_guest('service plc stop')
 323         else:
 324             self.run_in_host('service plc stop')
 325         return True
 326
 327     # could use a TestKey class
 328     def store_keys(self, options):
 329         for key_spec in self.plc_spec['keys']:
 330                 TestKey(self,key_spec).store_key()
 331         return True
 332
 333     def clean_keys(self, options):
 334         utils.system("rm -rf %s/keys/"%self.path)
 335
 336     def sites (self,options):
 337         return self.do_sites(options)
 338
 339     def clean_sites (self,options):
 340         return self.do_sites(options,action="delete")
 341
 342     def do_sites (self,options,action="add"):
 343         for site_spec in self.plc_spec['sites']:
 344             test_site = TestSite (self,site_spec)
 345             if (action != "add"):
 346                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 347                 test_site.delete_site()
 348                 # deleted with the site
 349                 #test_site.delete_users()
 350                 continue
 351             else:
 352                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 353                 test_site.create_site()
 354                 test_site.create_users()
 355         return True
 356
 357     def nodes (self, options):
 358         return self.do_nodes(options)
 359     def clean_nodes (self, options):
 360         return self.do_nodes(options,action="delete")
 361
 362     def do_nodes (self, options,action="add"):
 363         for site_spec in self.plc_spec['sites']:
 364             test_site = TestSite (self,site_spec)
 365             if action != "add":
 366                 utils.header("Deleting nodes in site %s"%test_site.name())
 367                 for node_spec in site_spec['nodes']:
 368                     test_node=TestNode(self,test_site,node_spec)
 369                     utils.header("Deleting %s"%test_node.name())
 370                     test_node.delete_node()
 371             else:
 372                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 373                 for node_spec in site_spec['nodes']:
 374                     utils.pprint('Creating node %s'%node_spec,node_spec)
 375                     test_node = TestNode (self,test_site,node_spec)
 376                     test_node.create_node ()
 377         return True
 378
 379     # create nodegroups if needed, and populate
 380     # no need for a clean_nodegroups if we are careful enough
 381     def nodegroups (self, options):
 382         # 1st pass to scan contents
 383         groups_dict = {}
 384         for site_spec in self.plc_spec['sites']:
 385             test_site = TestSite (self,site_spec)
 386             for node_spec in site_spec['nodes']:
 387                 test_node=TestNode (self,test_site,node_spec)
 388                 if node_spec.has_key('nodegroups'):
 389                     nodegroupnames=node_spec['nodegroups']
 390                     if isinstance(nodegroupnames,StringTypes):
 391                         nodegroupnames = [ nodegroupnames ]
 392                     for nodegroupname in nodegroupnames:
 393                         if not groups_dict.has_key(nodegroupname):
 394                             groups_dict[nodegroupname]=[]
 395                         groups_dict[nodegroupname].append(test_node.name())
 396         auth=self.auth_root()
 397         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 398             try:
 399                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 400             except:
 401                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 402             for node in group_nodes:
 403                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 404         return True
 405
 406     def all_hostnames (self) :
 407         hostnames = []
 408         for site_spec in self.plc_spec['sites']:
 409             hostnames += [ node_spec['node_fields']['hostname'] \
 410                            for node_spec in site_spec['nodes'] ]
 411         return hostnames
 412
 413     # gracetime : during the first <gracetime> minutes nothing gets printed
 414     def do_nodes_booted (self, minutes, gracetime=2):
 415         # compute timeout
 416         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 417         graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
 418         # the nodes that haven't checked yet - start with a full list and shrink over time
 419         tocheck = self.all_hostnames()
 420         utils.header("checking nodes %r"%tocheck)
 421         # create a dict hostname -> status
 422         status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
 423         while tocheck:
 424             # get their status
 425             tocheck_status=self.server.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
 426             # update status
 427             for array in tocheck_status:
 428                 hostname=array['hostname']
 429                 boot_state=array['boot_state']
 430                 if boot_state == 'boot':
 431                     utils.header ("%s has reached the 'boot' state"%hostname)
 432                 else:
 433                     # if it's a real node, never mind
 434                     (site_spec,node_spec)=self.locate_node(hostname)
 435                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 436                         utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
 437                         # let's cheat
 438                         boot_state = 'boot'
 439                     if datetime.datetime.now() > graceout:
 440                         utils.header ("%s still in '%s' state"%(hostname,boot_state))
 441                         graceout=datetime.datetime.now()+datetime.timedelta(1)
 442                 status[hostname] = boot_state
 443             # refresh tocheck
 444             tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
 445             if not tocheck:
 446                 return True
 447             if datetime.datetime.now() > timeout:
 448                 for hostname in tocheck:
 449                     utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
 450                 return False
 451             # otherwise, sleep for a while
 452             time.sleep(15)
 453         # only useful in empty plcs
 454         return True
 455
 456     def nodes_booted(self,options):
 457         return self.do_nodes_booted(minutes=5)
 458
 459     #to scan and store the nodes's public keys and avoid to ask for confirmation when  ssh
 460     def scan_publicKeys(self,hostnames):
 461         try:
 462             temp_knownhosts="/root/known_hosts"
 463             remote_knownhosts="/root/.ssh/known_hosts"
 464             self.run_in_host("touch %s"%temp_knownhosts )
 465             for hostname in hostnames:
 466                 utils.header("Scan Public %s key and store it in the known_host file(under the root image) "%hostname)
 467                 scan=self.run_in_host('ssh-keyscan -t rsa %s >> %s '%(hostname,temp_knownhosts))
 468             #Store the public keys in the right root image
 469             self.copy_in_guest(temp_knownhosts,remote_knownhosts,True)
 470             #clean the temp keys file used
 471             self.run_in_host('rm -f  %s '%temp_knownhosts )
 472         except Exception, err:
 473             print err
 474
 475     def do_check_nodesSsh(self,minutes):
 476         # compute timeout
 477         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 478         tocheck = self.all_hostnames()
 479         self.scan_publicKeys(tocheck)
 480         utils.header("checking Connectivity on nodes %r"%tocheck)
 481         while tocheck:
 482             for hostname in tocheck:
 483                 # try to ssh in nodes
 484                 access=self.run_in_guest('ssh -i /etc/planetlab/root_ssh_key.rsa root@%s date'%hostname )
 485                 if (not access):
 486                     utils.header('The node %s is sshable -->'%hostname)
 487                     # refresh tocheck
 488                     tocheck.remove(hostname)
 489                 else:
 490                     (site_spec,node_spec)=self.locate_node(hostname)
 491                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 492                         utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
 493                         tocheck.remove(hostname)
 494             if  not tocheck:
 495                 return True
 496             if datetime.datetime.now() > timeout:
 497                 for hostname in tocheck:
 498                     utils.header("FAILURE to ssh into %s"%hostname)
 499                 return False
 500             # otherwise, sleep for a while
 501             time.sleep(15)
 502         # only useful in empty plcs
 503         return True
 504
 505     def nodes_ssh(self, options):
 506         return  self.do_check_nodesSsh(minutes=2)
 507
 508     def bootcd (self, options):
 509         for site_spec in self.plc_spec['sites']:
 510             test_site = TestSite (self,site_spec)
 511             for node_spec in site_spec['nodes']:
 512                 test_node=TestNode (self,test_site,node_spec)
 513                 test_node.prepare_area()
 514                 test_node.create_boot_cd()
 515                 test_node.configure_qemu()
 516         return True
 517
 518     def do_check_intiscripts(self):
 519         for site_spec in self.plc_spec['sites']:
 520                 test_site = TestSite (self,site_spec)
 521                 test_node = TestNode (self,test_site,site_spec['nodes'])
 522                 for slice_spec in self.plc_spec['slices']:
 523                         test_slice=TestSlice (self,test_site,slice_spec)
 524                         test_sliver=TestSliver(self,test_node,test_slice)
 525                         init_status=test_sliver.get_initscript(slice_spec)
 526                         if (not init_status):
 527                                 return False
 528                 return init_status
 529
 530     def check_initscripts(self, options):
 531             return self.do_check_intiscripts()
 532
 533     def initscripts (self, options):
 534         for initscript in self.plc_spec['initscripts']:
 535             utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 536             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 537         return True
 538
 539     def slices (self, options):
 540         return self.do_slices()
 541
 542     def clean_slices (self, options):
 543         return self.do_slices("delete")
 544
 545     def do_slices (self,  action="add"):
 546         for slice in self.plc_spec['slices']:
 547             site_spec = self.locate_site (slice['sitename'])
 548             test_site = TestSite(self,site_spec)
 549             test_slice=TestSlice(self,test_site,slice)
 550             if action != "add":
 551                 utils.header("Deleting slices in site %s"%test_site.name())
 552                 test_slice.delete_slice()
 553             else:
 554                 utils.pprint("Creating slice",slice)
 555                 test_slice.create_slice()
 556                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
 557         return True
 558
 559     def check_slices(self, options):
 560         for slice_spec in self.plc_spec['slices']:
 561             site_spec = self.locate_site (slice_spec['sitename'])
 562             test_site = TestSite(self,site_spec)
 563             test_slice=TestSlice(self,test_site,slice_spec)
 564             status=test_slice.do_check_slice(options)
 565             if (not status):
 566                 return False
 567         return status
 568
 569     def start_nodes (self, options):
 570         utils.header("Starting  nodes")
 571         for site_spec in self.plc_spec['sites']:
 572             TestSite(self,site_spec).start_nodes (options)
 573         return True
 574
 575     def stop_nodes (self, options):
 576         self.kill_all_qemus(options)
 577         return True
 578
 579     def check_tcp (self, options):
 580             #we just need to create a sliver object nothing else
 581             test_sliver=TestSliver(self,
 582                                    TestNode(self, TestSite(self,self.plc_spec['sites'][0]),
 583                                             self.plc_spec['sites'][0]['nodes'][0]),
 584                                    TestSlice(self,TestSite(self,self.plc_spec['sites'][0]),
 585                                              self.plc_spec['slices']))
 586             return test_sliver.do_check_tcp(self.plc_spec['tcp_param'],options)
 587
 588     # returns the filename to use for sql dump/restore, using options.dbname if set
 589     def dbfile (self, database, options):
 590         # uses options.dbname if it is found
 591         try:
 592             name=options.dbname
 593             if not isinstance(name,StringTypes):
 594                 raise Exception
 595         except:
 596             t=datetime.datetime.now()
 597             d=t.date()
 598             name=str(d)
 599         return "/root/%s-%s.sql"%(database,name)
 600
 601     def db_dump(self, options):
 602
 603         dump=self.dbfile("planetab4",options)
 604         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 605         utils.header('Dumped planetlab4 database in %s'%dump)
 606         return True
 607
 608     def db_restore(self, options):
 609         dump=self.dbfile("planetab4",options)
 610         ##stop httpd service
 611         self.run_in_guest('service httpd stop')
 612         # xxx - need another wrapper
 613         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 614         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 615         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 616         ##starting httpd service
 617         self.run_in_guest('service httpd start')
 618
 619         utils.header('Database restored from ' + dump)
 620
 621     @standby_generic
 622     def standby_1(): pass
 623     @standby_generic
 624     def standby_2(): pass
 625     @standby_generic
 626     def standby_3(): pass
 627     @standby_generic
 628     def standby_4(): pass
 629     @standby_generic
 630     def standby_5(): pass
 631     @standby_generic
 632     def standby_6(): pass
 633     @standby_generic
 634     def standby_7(): pass
 635     @standby_generic
 636     def standby_8(): pass
 637     @standby_generic
 638     def standby_9(): pass
 639     @standby_generic
 640     def standby_10(): pass
 641     @standby_generic
 642     def standby_11(): pass
 643     @standby_generic
 644     def standby_12(): pass
 645     @standby_generic
 646     def standby_13(): pass
 647     @standby_generic
 648     def standby_14(): pass
 649     @standby_generic
 650     def standby_15(): pass
 651     @standby_generic
 652     def standby_16(): pass
 653     @standby_generic
 654     def standby_17(): pass
 655     @standby_generic
 656     def standby_18(): pass
 657     @standby_generic
 658     def standby_19(): pass
 659     @standby_generic
 660     def standby_20(): pass
 661