system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16 from TestSlice import TestSlice
  17 from TestSliver import TestSliver
  18 from TestBox import TestBox
  19
  20 # step methods must take (self, options) and return a boolean
  21
  22 def standby(minutes):
  23         utils.header('Entering StandBy for %d mn'%minutes)
  24         time.sleep(60*minutes)
  25         return True
  26
  27 def standby_generic (func):
  28     def actual(self,options):
  29         minutes=int(func.__name__.split("_")[1])
  30         return standby(minutes)
  31     return actual
  32
  33 class TestPlc:
  34
  35     def __init__ (self,plc_spec):
  36         self.plc_spec=plc_spec
  37         self.path=os.path.dirname(sys.argv[0])
  38         try:
  39             self.vserverip=plc_spec['vserverip']
  40             self.vservername=plc_spec['vservername']
  41             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  42             self.vserver=True
  43         except:
  44             self.vserver=False
  45             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  46         utils.header('Using API url %s'%self.url)
  47         self.server=xmlrpclib.Server(self.url,allow_none=True)
  48
  49     def name(self):
  50         name=self.plc_spec['name']
  51         if self.vserver:
  52             return name+"[%s]"%self.vservername
  53         else:
  54             return name+"[chroot]"
  55
  56     def hostname(self):
  57         return self.plc_spec['hostname']
  58
  59     def is_local (self):
  60         return utils.is_local(self.hostname())
  61
  62     # define the API methods on this object through xmlrpc
  63     # would help, but not strictly necessary
  64     def connect (self):
  65         pass
  66
  67     # command gets run in the chroot/vserver
  68     def host_to_guest(self,command):
  69         if self.vserver:
  70             return "vserver %s exec %s"%(self.vservername,command)
  71         else:
  72             return "chroot /plc/root %s"%utils.backslash_shell_specials(command)
  73
  74     # command gets run on the right box
  75     def to_host(self,command):
  76         if self.is_local():
  77             return command
  78         else:
  79             return "ssh %s %s"%(self.hostname(),utils.backslash_shell_specials(command))
  80
  81     def full_command(self,command):
  82         return self.to_host(self.host_to_guest(command))
  83
  84     def run_in_guest (self,command):
  85         return utils.system(self.full_command(command))
  86     def run_in_host (self,command):
  87         return utils.system(self.to_host(command))
  88
  89     # xxx quick n dirty
  90     def run_in_guest_piped (self,local,remote):
  91         return utils.system(local+" | "+self.full_command(remote))
  92
  93     # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
  94     def copy_in_guest (self, localfile, remotefile, in_data=False):
  95         if in_data:
  96             chroot_dest="/plc/data"
  97         else:
  98             chroot_dest="/plc/root"
  99         if self.is_local():
 100             if not self.vserver:
 101                 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
 102             else:
 103                 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
 104         else:
 105             if not self.vserver:
 106                 utils.system("scp %s %s:%s/%s"%(localfile,self.hostname(),chroot_dest,remotefile))
 107             else:
 108                 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.hostname(),self.vservername,remotefile))
 109
 110     def auth_root (self):
 111         return {'Username':self.plc_spec['PLC_ROOT_USER'],
 112                 'AuthMethod':'password',
 113                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
 114                 'Role' : self.plc_spec['role']
 115                 }
 116     def locate_site (self,sitename):
 117         for site in self.plc_spec['sites']:
 118             if site['site_fields']['name'] == sitename:
 119                 return site
 120             if site['site_fields']['login_base'] == sitename:
 121                 return site
 122         raise Exception,"Cannot locate site %s"%sitename
 123
 124     def locate_node (self,nodename):
 125         for site in self.plc_spec['sites']:
 126             for node in site['nodes']:
 127                 if node['node_fields']['hostname'] == nodename:
 128                     return (site,node)
 129         raise Exception,"Cannot locate node %s"%nodename
 130
 131     def locate_key (self,keyname):
 132         for key in self.plc_spec['keys']:
 133             if key['name'] == keyname:
 134                 return key
 135         raise Exception,"Cannot locate key %s"%keyname
 136
 137     # all different hostboxes used in this plc
 138     def gather_hostBoxes(self):
 139         # maps on sites and nodes, return [ (host_box,test_node) ]
 140         tuples=[]
 141         for site_spec in self.plc_spec['sites']:
 142             test_site = TestSite (self,site_spec)
 143             for node_spec in site_spec['nodes']:
 144                 test_node = TestNode (self, test_site, node_spec)
 145                 if not test_node.is_real():
 146                     tuples.append( (test_node.host_box(),test_node) )
 147         # transform into a dict { 'host_box' -> [ hostnames .. ] }
 148         result = {}
 149         for (box,node) in tuples:
 150             if not result.has_key(box):
 151                 result[box]=[node]
 152             else:
 153                 result[box].append(node)
 154         return result
 155
 156     # a step for checking this stuff
 157     def showboxes (self,options):
 158         print 'showboxes'
 159         for (box,nodes) in self.gather_hostBoxes().iteritems():
 160             print box,":"," + ".join( [ node.name() for node in nodes ] )
 161         return True
 162
 163     # make this a valid step
 164     def kill_all_qemus(self,options):
 165         for (box,nodes) in self.gather_hostBoxes().iteritems():
 166             # this is the brute force version, kill all qemus on that host box
 167             TestBox(box,options.buildname).kill_all_qemus()
 168         return True
 169
 170     # make this a valid step
 171     def list_all_qemus(self,options):
 172         for (box,nodes) in self.gather_hostBoxes().iteritems():
 173             # push the script
 174             TestBox(box,options.buildname).copy("qemu_kill.sh")
 175             # this is the brute force version, kill all qemus on that host box
 176             TestBox(box,options.buildname).run_in_buildname("qemu_kill.sh -l")
 177         return True
 178
 179     # kill only the right qemus
 180     def kill_qemus(self,options):
 181         for (box,nodes) in self.gather_hostBoxes().iteritems():
 182             # push the script
 183             TestBox(box,options.buildname).copy("qemu_kill.sh")
 184             # the fine-grain version
 185             for node in nodes:
 186                 node.kill_qemu()
 187         return True
 188
 189     def clear_ssh_config (self,options):
 190         # install local ssh_config file as root's .ssh/config - ssh should be quiet
 191         # dir might need creation first
 192         self.run_in_guest("mkdir /root/.ssh")
 193         self.run_in_guest("chmod 700 /root/.ssh")
 194         # this does not work - > redirection somehow makes it until an argument to cat
 195         #self.run_in_guest_piped("cat ssh_config","cat > /root/.ssh/config")
 196         self.copy_in_guest("ssh_config","/root/.ssh/config",True)
 197         return True
 198
 199     #################### step methods
 200
 201     ### uninstall
 202     def uninstall_chroot(self,options):
 203         self.run_in_host('service plc safestop')
 204         #####detecting the last myplc version installed and remove it
 205         self.run_in_host('rpm -e myplc')
 206         ##### Clean up the /plc directory
 207         self.run_in_host('rm -rf  /plc/data')
 208         ##### stop any running vservers
 209         self.run_in_host('for vserver in $(ls /vservers/* | sed -e s,/vservers/,,) ; do vserver $vserver stop ; done')
 210         return True
 211
 212     def uninstall_vserver(self,options):
 213         self.run_in_host("vserver --silent %s delete"%self.vservername)
 214         return True
 215
 216     def uninstall(self,options):
 217         # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
 218         # it sounds safer to have the former uninstalled too
 219         # now the vserver method cannot be invoked for chroot instances as vservername is required
 220         if self.vserver:
 221             self.uninstall_vserver(options)
 222             self.uninstall_chroot(options)
 223         else:
 224             self.uninstall_chroot(options)
 225         return True
 226
 227     ### install
 228     def install_chroot(self,options):
 229         # nothing to do
 230         return True
 231
 232     # xxx this would not work with hostname != localhost as mylc-init-vserver was extracted locally
 233     def install_vserver(self,options):
 234         # we need build dir for vtest-init-vserver
 235         if self.is_local():
 236             # a full path for the local calls
 237             build_dir=self.path+"/build"
 238         else:
 239             # use a standard name - will be relative to HOME
 240             build_dir="tests-system-build"
 241         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 242         if self.run_in_host(build_checkout) != 0:
 243             raise Exception,"Cannot checkout build dir"
 244         # the repo url is taken from myplc-url
 245         # with the last two steps (i386/myplc...) removed
 246         repo_url = options.myplc_url
 247         repo_url = os.path.dirname(repo_url)
 248         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 249             (build_dir,self.vservername,repo_url,self.vserverip)
 250         if self.run_in_host(create_vserver) != 0:
 251             raise Exception,"Could not create vserver for %s"%self.vservername
 252         return True
 253
 254     def install(self,options):
 255         if self.vserver:
 256             return self.install_vserver(options)
 257         else:
 258             return self.install_chroot(options)
 259
 260     ### install_rpm
 261     def cache_rpm(self,url):
 262         self.run_in_host('rm -rf *.rpm')
 263         utils.header('Curling rpm from %s'%url)
 264         id= self.run_in_host('curl -O '+url)
 265         if (id != 0):
 266                 raise Exception,"Could not get rpm from  %s"%url
 267                 return False
 268         return True
 269
 270     def install_rpm_chroot(self,options):
 271         rpm = os.path.basename(options.myplc_url)
 272         if (not os.path.isfile(rpm)):
 273                 self.cache_rpm(options.myplc_url)
 274         utils.header('Installing the :  %s'%rpm)
 275         self.run_in_host('rpm -Uvh '+rpm)
 276         self.run_in_host('service plc mount')
 277         return True
 278
 279     def install_rpm_vserver(self,options):
 280         self.run_in_guest("yum -y install myplc-native")
 281         return True
 282
 283     def install_rpm(self,options):
 284         if self.vserver:
 285             return self.install_rpm_vserver(options)
 286         else:
 287             return self.install_rpm_chroot(options)
 288
 289     ###
 290     def configure(self,options):
 291         tmpname='%s.plc-config-tty'%(self.name())
 292         fileconf=open(tmpname,'w')
 293         for var in [ 'PLC_NAME',
 294                      'PLC_ROOT_PASSWORD',
 295                      'PLC_ROOT_USER',
 296                      'PLC_MAIL_ENABLED',
 297                      'PLC_MAIL_SUPPORT_ADDRESS',
 298                      'PLC_DB_HOST',
 299                      'PLC_API_HOST',
 300                      'PLC_WWW_HOST',
 301                      'PLC_BOOT_HOST',
 302                      'PLC_NET_DNS1',
 303                      'PLC_NET_DNS2']:
 304             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 305         fileconf.write('w\n')
 306         fileconf.write('q\n')
 307         fileconf.close()
 308         utils.system('cat %s'%tmpname)
 309         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 310         utils.system('rm %s'%tmpname)
 311         return True
 312
 313     # the chroot install is slightly different to this respect
 314     def start(self, options):
 315         if self.vserver:
 316             self.run_in_guest('service plc start')
 317         else:
 318             self.run_in_host('service plc start')
 319         return True
 320
 321     def stop(self, options):
 322         if self.vserver:
 323             self.run_in_guest('service plc stop')
 324         else:
 325             self.run_in_host('service plc stop')
 326         return True
 327
 328     # could use a TestKey class
 329     def store_keys(self, options):
 330         for key_spec in self.plc_spec['keys']:
 331             TestKey(self,key_spec).store_key()
 332         return True
 333
 334     def clean_keys(self, options):
 335         utils.system("rm -rf %s/keys/"%self.path)
 336
 337     def sites (self,options):
 338         return self.do_sites(options)
 339
 340     def clean_sites (self,options):
 341         return self.do_sites(options,action="delete")
 342
 343     def do_sites (self,options,action="add"):
 344         for site_spec in self.plc_spec['sites']:
 345             test_site = TestSite (self,site_spec)
 346             if (action != "add"):
 347                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 348                 test_site.delete_site()
 349                 # deleted with the site
 350                 #test_site.delete_users()
 351                 continue
 352             else:
 353                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 354                 test_site.create_site()
 355                 test_site.create_users()
 356         return True
 357
 358     def nodes (self, options):
 359         return self.do_nodes(options)
 360     def clean_nodes (self, options):
 361         return self.do_nodes(options,action="delete")
 362
 363     def do_nodes (self, options,action="add"):
 364         for site_spec in self.plc_spec['sites']:
 365             test_site = TestSite (self,site_spec)
 366             if action != "add":
 367                 utils.header("Deleting nodes in site %s"%test_site.name())
 368                 for node_spec in site_spec['nodes']:
 369                     test_node=TestNode(self,test_site,node_spec)
 370                     utils.header("Deleting %s"%test_node.name())
 371                     test_node.delete_node()
 372             else:
 373                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 374                 for node_spec in site_spec['nodes']:
 375                     utils.pprint('Creating node %s'%node_spec,node_spec)
 376                     test_node = TestNode (self,test_site,node_spec)
 377                     test_node.create_node ()
 378         return True
 379
 380     # create nodegroups if needed, and populate
 381     # no need for a clean_nodegroups if we are careful enough
 382     def nodegroups (self, options):
 383         # 1st pass to scan contents
 384         groups_dict = {}
 385         for site_spec in self.plc_spec['sites']:
 386             test_site = TestSite (self,site_spec)
 387             for node_spec in site_spec['nodes']:
 388                 test_node=TestNode (self,test_site,node_spec)
 389                 if node_spec.has_key('nodegroups'):
 390                     nodegroupnames=node_spec['nodegroups']
 391                     if isinstance(nodegroupnames,StringTypes):
 392                         nodegroupnames = [ nodegroupnames ]
 393                     for nodegroupname in nodegroupnames:
 394                         if not groups_dict.has_key(nodegroupname):
 395                             groups_dict[nodegroupname]=[]
 396                         groups_dict[nodegroupname].append(test_node.name())
 397         auth=self.auth_root()
 398         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 399             try:
 400                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 401             except:
 402                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 403             for node in group_nodes:
 404                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 405         return True
 406
 407     def all_hostnames (self) :
 408         hostnames = []
 409         for site_spec in self.plc_spec['sites']:
 410             hostnames += [ node_spec['node_fields']['hostname'] \
 411                            for node_spec in site_spec['nodes'] ]
 412         return hostnames
 413
 414     # gracetime : during the first <gracetime> minutes nothing gets printed
 415     def do_nodes_booted (self, minutes, gracetime=2):
 416         # compute timeout
 417         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 418         graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
 419         # the nodes that haven't checked yet - start with a full list and shrink over time
 420         tocheck = self.all_hostnames()
 421         utils.header("checking nodes %r"%tocheck)
 422         # create a dict hostname -> status
 423         status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
 424         while tocheck:
 425             # get their status
 426             tocheck_status=self.server.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
 427             # update status
 428             for array in tocheck_status:
 429                 hostname=array['hostname']
 430                 boot_state=array['boot_state']
 431                 if boot_state == 'boot':
 432                     utils.header ("%s has reached the 'boot' state"%hostname)
 433                 else:
 434                     # if it's a real node, never mind
 435                     (site_spec,node_spec)=self.locate_node(hostname)
 436                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 437                         utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
 438                         # let's cheat
 439                         boot_state = 'boot'
 440                     if datetime.datetime.now() > graceout:
 441                         utils.header ("%s still in '%s' state"%(hostname,boot_state))
 442                         graceout=datetime.datetime.now()+datetime.timedelta(1)
 443                 status[hostname] = boot_state
 444             # refresh tocheck
 445             tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
 446             if not tocheck:
 447                 return True
 448             if datetime.datetime.now() > timeout:
 449                 for hostname in tocheck:
 450                     utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
 451                 return False
 452             # otherwise, sleep for a while
 453             time.sleep(15)
 454         # only useful in empty plcs
 455         return True
 456
 457     def nodes_booted(self,options):
 458         return self.do_nodes_booted(minutes=5)
 459
 460     #to scan and store the nodes's public keys and avoid to ask for confirmation when  ssh
 461     def scan_publicKeys(self,hostnames):
 462         try:
 463             temp_knownhosts="/root/known_hosts"
 464             remote_knownhosts="/root/.ssh/known_hosts"
 465             self.run_in_host("touch %s"%temp_knownhosts )
 466             for hostname in hostnames:
 467                 utils.header("Scan Public %s key and store it in the known_host file(under the root image) "%hostname)
 468                 scan=self.run_in_host('ssh-keyscan -t rsa %s >> %s '%(hostname,temp_knownhosts))
 469             #Store the public keys in the right root image
 470             self.copy_in_guest(temp_knownhosts,remote_knownhosts,True)
 471             #clean the temp keys file used
 472             self.run_in_host('rm -f  %s '%temp_knownhosts )
 473         except Exception, err:
 474             print err
 475
 476     def do_check_nodesSsh(self,minutes):
 477         # compute timeout
 478         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 479         tocheck = self.all_hostnames()
 480         self.scan_publicKeys(tocheck)
 481         utils.header("checking Connectivity on nodes %r"%tocheck)
 482         while tocheck:
 483             for hostname in tocheck:
 484                 # try to ssh in nodes
 485                 access=self.run_in_guest('ssh -i /etc/planetlab/root_ssh_key.rsa root@%s date'%hostname )
 486                 if (not access):
 487                     utils.header('The node %s is sshable -->'%hostname)
 488                     # refresh tocheck
 489                     tocheck.remove(hostname)
 490                 else:
 491                     (site_spec,node_spec)=self.locate_node(hostname)
 492                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 493                         utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
 494                         tocheck.remove(hostname)
 495             if  not tocheck:
 496                 return True
 497             if datetime.datetime.now() > timeout:
 498                 for hostname in tocheck:
 499                     utils.header("FAILURE to ssh into %s"%hostname)
 500                 return False
 501             # otherwise, sleep for a while
 502             time.sleep(15)
 503         # only useful in empty plcs
 504         return True
 505
 506     def nodes_ssh(self, options):
 507         return  self.do_check_nodesSsh(minutes=2)
 508
 509     def bootcd (self, options):
 510         for site_spec in self.plc_spec['sites']:
 511             test_site = TestSite (self,site_spec)
 512             for node_spec in site_spec['nodes']:
 513                 test_node=TestNode (self,test_site,node_spec)
 514                 test_node.prepare_area()
 515                 test_node.create_boot_cd()
 516                 test_node.configure_qemu()
 517         return True
 518
 519     def do_check_intiscripts(self):
 520         for site_spec in self.plc_spec['sites']:
 521                 test_site = TestSite (self,site_spec)
 522                 test_node = TestNode (self,test_site,site_spec['nodes'])
 523                 for slice_spec in self.plc_spec['slices']:
 524                         test_slice=TestSlice (self,test_site,slice_spec)
 525                         test_sliver=TestSliver(self,test_node,test_slice)
 526                         init_status=test_sliver.get_initscript(slice_spec)
 527                         if (not init_status):
 528                                 return False
 529                 return init_status
 530
 531     def check_initscripts(self, options):
 532             return self.do_check_intiscripts()
 533
 534     def initscripts (self, options):
 535         for initscript in self.plc_spec['initscripts']:
 536             utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 537             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 538         return True
 539
 540     def slices (self, options):
 541         return self.do_slices()
 542
 543     def clean_slices (self, options):
 544         return self.do_slices("delete")
 545
 546     def do_slices (self,  action="add"):
 547         for slice in self.plc_spec['slices']:
 548             site_spec = self.locate_site (slice['sitename'])
 549             test_site = TestSite(self,site_spec)
 550             test_slice=TestSlice(self,test_site,slice)
 551             if action != "add":
 552                 utils.header("Deleting slices in site %s"%test_site.name())
 553                 test_slice.delete_slice()
 554             else:
 555                 utils.pprint("Creating slice",slice)
 556                 test_slice.create_slice()
 557                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
 558         return True
 559
 560     def check_slices(self, options):
 561         for slice_spec in self.plc_spec['slices']:
 562             site_spec = self.locate_site (slice_spec['sitename'])
 563             test_site = TestSite(self,site_spec)
 564             test_slice=TestSlice(self,test_site,slice_spec)
 565             status=test_slice.do_check_slice(options)
 566             if (not status):
 567                 return False
 568         return status
 569
 570     def start_nodes (self, options):
 571         utils.header("Starting  nodes")
 572         for site_spec in self.plc_spec['sites']:
 573             TestSite(self,site_spec).start_nodes (options)
 574         return True
 575
 576     def stop_nodes (self, options):
 577         self.kill_all_qemus(options)
 578         return True
 579
 580     def check_tcp (self, options):
 581             #we just need to create a sliver object nothing else
 582             test_sliver=TestSliver(self,
 583                                    TestNode(self, TestSite(self,self.plc_spec['sites'][0]),
 584                                             self.plc_spec['sites'][0]['nodes'][0]),
 585                                    TestSlice(self,TestSite(self,self.plc_spec['sites'][0]),
 586                                              self.plc_spec['slices']))
 587             return test_sliver.do_check_tcp(self.plc_spec['tcp_param'],options)
 588
 589     # returns the filename to use for sql dump/restore, using options.dbname if set
 590     def dbfile (self, database, options):
 591         # uses options.dbname if it is found
 592         try:
 593             name=options.dbname
 594             if not isinstance(name,StringTypes):
 595                 raise Exception
 596         except:
 597             t=datetime.datetime.now()
 598             d=t.date()
 599             name=str(d)
 600         return "/root/%s-%s.sql"%(database,name)
 601
 602     def db_dump(self, options):
 603
 604         dump=self.dbfile("planetab4",options)
 605         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 606         utils.header('Dumped planetlab4 database in %s'%dump)
 607         return True
 608
 609     def db_restore(self, options):
 610         dump=self.dbfile("planetab4",options)
 611         ##stop httpd service
 612         self.run_in_guest('service httpd stop')
 613         # xxx - need another wrapper
 614         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 615         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 616         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 617         ##starting httpd service
 618         self.run_in_guest('service httpd start')
 619
 620         utils.header('Database restored from ' + dump)
 621
 622     @standby_generic
 623     def standby_1(): pass
 624     @standby_generic
 625     def standby_2(): pass
 626     @standby_generic
 627     def standby_3(): pass
 628     @standby_generic
 629     def standby_4(): pass
 630     @standby_generic
 631     def standby_5(): pass
 632     @standby_generic
 633     def standby_6(): pass
 634     @standby_generic
 635     def standby_7(): pass
 636     @standby_generic
 637     def standby_8(): pass
 638     @standby_generic
 639     def standby_9(): pass
 640     @standby_generic
 641     def standby_10(): pass
 642     @standby_generic
 643     def standby_11(): pass
 644     @standby_generic
 645     def standby_12(): pass
 646     @standby_generic
 647     def standby_13(): pass
 648     @standby_generic
 649     def standby_14(): pass
 650     @standby_generic
 651     def standby_15(): pass
 652     @standby_generic
 653     def standby_16(): pass
 654     @standby_generic
 655     def standby_17(): pass
 656     @standby_generic
 657     def standby_18(): pass
 658     @standby_generic
 659     def standby_19(): pass
 660     @standby_generic
 661     def standby_20(): pass
 662