system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16 from TestSlice import TestSlice
  17 from TestSliver import TestSliver
  18 from TestBox import TestBox
  19 from TestSsh import TestSsh
  20
  21 # step methods must take (self, options) and return a boolean
  22
  23 def standby(minutes):
  24         utils.header('Entering StandBy for %d mn'%minutes)
  25         time.sleep(60*minutes)
  26         return True
  27
  28 def standby_generic (func):
  29     def actual(self,options):
  30         minutes=int(func.__name__.split("_")[1])
  31         return standby(minutes)
  32     return actual
  33
  34 class TestPlc:
  35
  36     def __init__ (self,plc_spec):
  37         self.plc_spec=plc_spec
  38         self.path=os.path.dirname(sys.argv[0])
  39         self.test_ssh=TestSsh(self)
  40         try:
  41             self.vserverip=plc_spec['vserverip']
  42             self.vservername=plc_spec['vservername']
  43             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  44             self.vserver=True
  45         except:
  46             self.vserver=False
  47             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  48         utils.header('Using API url %s'%self.url)
  49         self.server=xmlrpclib.Server(self.url,allow_none=True)
  50
  51     def name(self):
  52         name=self.plc_spec['name']
  53         if self.vserver:
  54             return name+"[%s]"%self.vservername
  55         else:
  56             return name+"[chroot]"
  57
  58     def hostname(self):
  59         return self.plc_spec['hostname']
  60
  61     def is_local (self):
  62         return self.test_ssh.is_local()
  63
  64     # define the API methods on this object through xmlrpc
  65     # would help, but not strictly necessary
  66     def connect (self):
  67         pass
  68
  69     #command gets run in the chroot/vserver
  70     def host_to_guest(self,command):
  71         if self.vserver:
  72             return "vserver %s exec %s"%(self.vservername,command)
  73         else:
  74             return "chroot /plc/root %s"%TestSsh.backslash_shell_specials(command)
  75
  76     # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
  77     def copy_in_guest (self, localfile, remotefile, in_data=False):
  78         if in_data:
  79             chroot_dest="/plc/data"
  80         else:
  81             chroot_dest="/plc/root"
  82         if self.is_local():
  83             if not self.vserver:
  84                 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
  85             else:
  86                 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
  87         else:
  88             if not self.vserver:
  89                 utils.system("scp %s %s:%s/%s"%(localfile,self.hostname(),chroot_dest,remotefile))
  90             else:
  91                 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.hostname(),self.vservername,remotefile))
  92
  93     def auth_root (self):
  94         return {'Username':self.plc_spec['PLC_ROOT_USER'],
  95                 'AuthMethod':'password',
  96                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
  97                 'Role' : self.plc_spec['role']
  98                 }
  99     def locate_site (self,sitename):
 100         for site in self.plc_spec['sites']:
 101             if site['site_fields']['name'] == sitename:
 102                 return site
 103             if site['site_fields']['login_base'] == sitename:
 104                 return site
 105         raise Exception,"Cannot locate site %s"%sitename
 106
 107     def locate_node (self,nodename):
 108         for site in self.plc_spec['sites']:
 109             for node in site['nodes']:
 110                 if node['node_fields']['hostname'] == nodename:
 111                     return (site,node)
 112         raise Exception,"Cannot locate node %s"%nodename
 113
 114     def locate_key (self,keyname):
 115         for key in self.plc_spec['keys']:
 116             if key['name'] == keyname:
 117                 return key
 118         raise Exception,"Cannot locate key %s"%keyname
 119
 120     # all different hostboxes used in this plc
 121     def gather_hostBoxes(self):
 122         # maps on sites and nodes, return [ (host_box,test_node) ]
 123         tuples=[]
 124         for site_spec in self.plc_spec['sites']:
 125             test_site = TestSite (self,site_spec)
 126             for node_spec in site_spec['nodes']:
 127                 test_node = TestNode (self, test_site, node_spec)
 128                 if not test_node.is_real():
 129                     tuples.append( (test_node.host_box(),test_node) )
 130         # transform into a dict { 'host_box' -> [ hostnames .. ] }
 131         result = {}
 132         for (box,node) in tuples:
 133             if not result.has_key(box):
 134                 result[box]=[node]
 135             else:
 136                 result[box].append(node)
 137         return result
 138
 139     # a step for checking this stuff
 140     def showboxes (self,options):
 141         print 'showboxes'
 142         for (box,nodes) in self.gather_hostBoxes().iteritems():
 143             print box,":"," + ".join( [ node.name() for node in nodes ] )
 144         return True
 145
 146     # make this a valid step
 147     def kill_all_qemus(self,options):
 148         for (box,nodes) in self.gather_hostBoxes().iteritems():
 149             # this is the brute force version, kill all qemus on that host box
 150             TestBox(box,options.buildname).kill_all_qemus()
 151         return True
 152
 153     # make this a valid step
 154     def list_all_qemus(self,options):
 155         for (box,nodes) in self.gather_hostBoxes().iteritems():
 156             # push the script
 157             TestBox(box,options.buildname).copy("qemu_kill.sh")
 158             # this is the brute force version, kill all qemus on that host box
 159             TestBox(box,options.buildname).run_in_buildname("qemu_kill.sh -l")
 160         return True
 161
 162     # kill only the right qemus
 163     def force_kill_qemus(self,options):
 164         for (box,nodes) in self.gather_hostBoxes().iteritems():
 165             # push the script
 166             TestBox(box,options.buildname).copy("qemu_kill.sh")
 167             # the fine-grain version
 168             for node in nodes:
 169                 node.kill_qemu()
 170         return True
 171
 172     def clear_ssh_config (self,options):
 173         # install local ssh_config file as root's .ssh/config - ssh should be quiet
 174         # dir might need creation first
 175         self.test_ssh.run_in_guest("mkdir /root/.ssh")
 176         self.test_ssh.run_in_guest("chmod 700 /root/.ssh")
 177         # this does not work - > redirection somehow makes it until an argument to cat
 178         #self.run_in_guest_piped("cat ssh_config","cat > /root/.ssh/config")
 179         self.copy_in_guest("ssh_config","/root/.ssh/config",True)
 180         return True
 181
 182     #################### step methods
 183
 184     ### uninstall
 185     def uninstall_chroot(self,options):
 186         self.test_ssh.run_in_host('service plc safestop')
 187         #####detecting the last myplc version installed and remove it
 188         self.test_ssh.run_in_host('rpm -e myplc')
 189         ##### Clean up the /plc directory
 190         self.test_ssh.run_in_host('rm -rf  /plc/data')
 191         ##### stop any running vservers
 192         self.test_ssh.run_in_host('for vserver in $(ls /vservers/* | sed -e s,/vservers/,,) ; do vserver $vserver stop ; done')
 193         return True
 194
 195     def uninstall_vserver(self,options):
 196         self.test_ssh.run_in_host("vserver --silent %s delete"%self.vservername)
 197         return True
 198
 199     def uninstall(self,options):
 200         # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
 201         # it sounds safer to have the former uninstalled too
 202         # now the vserver method cannot be invoked for chroot instances as vservername is required
 203         if self.vserver:
 204             self.uninstall_vserver(options)
 205             self.uninstall_chroot(options)
 206         else:
 207             self.uninstall_chroot(options)
 208         return True
 209
 210     ### install
 211     def install_chroot(self,options):
 212         # nothing to do
 213         return True
 214
 215     def install_vserver(self,options):
 216         # we need build dir for vtest-init-vserver
 217         if self.is_local():
 218             # a full path for the local calls
 219             build_dir=self.path+"/build"
 220         else:
 221             # use a standard name - will be relative to HOME
 222             build_dir="options.buildname"
 223         # run checkout in any case - would do an update if already exists
 224         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 225         if self.test_ssh.run_in_host(build_checkout) != 0:
 226             raise Exception,"Cannot checkout build dir"
 227         # the repo url is taken from myplc-url
 228         # with the last two steps (i386/myplc...) removed
 229         repo_url = options.myplc_url
 230         for level in [ 'rpmname','arch' ]:
 231             repo_url = os.path.dirname(repo_url)
 232         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 233             (build_dir,self.vservername,repo_url,self.vserverip)
 234         if self.test_ssh.run_in_host(create_vserver) != 0:
 235             raise Exception,"Could not create vserver for %s"%self.vservername
 236         return True
 237
 238     def install(self,options):
 239         if self.vserver:
 240             return self.install_vserver(options)
 241         else:
 242             return self.install_chroot(options)
 243
 244     ### install_rpm
 245     def cache_rpm(self,url):
 246         self.test_ssh.run_in_host('rm -rf *.rpm')
 247         utils.header('Curling rpm from %s'%url)
 248         id= self.test_ssh.run_in_host('curl -O '+url)
 249         if (id != 0):
 250                 raise Exception,"Could not get rpm from  %s"%url
 251                 return False
 252         return True
 253
 254     def install_rpm_chroot(self,options):
 255         rpm = os.path.basename(options.myplc_url)
 256         if (not os.path.isfile(rpm)):
 257                 self.cache_rpm(options.myplc_url)
 258         utils.header('Installing the :  %s'%rpm)
 259         self.test_ssh.run_in_host('rpm -Uvh '+rpm)
 260         self.test_ssh.run_in_host('service plc mount')
 261         return True
 262
 263     def install_rpm_vserver(self,options):
 264         self.test_ssh.run_in_guest("yum -y install myplc-native")
 265         return True
 266
 267     def install_rpm(self,options):
 268         if self.vserver:
 269             return self.install_rpm_vserver(options)
 270         else:
 271             return self.install_rpm_chroot(options)
 272
 273     ###
 274     def configure(self,options):
 275         tmpname='%s.plc-config-tty'%(self.name())
 276         fileconf=open(tmpname,'w')
 277         for var in [ 'PLC_NAME',
 278                      'PLC_ROOT_PASSWORD',
 279                      'PLC_ROOT_USER',
 280                      'PLC_MAIL_ENABLED',
 281                      'PLC_MAIL_SUPPORT_ADDRESS',
 282                      'PLC_DB_HOST',
 283                      'PLC_API_HOST',
 284                      'PLC_WWW_HOST',
 285                      'PLC_BOOT_HOST',
 286                      'PLC_NET_DNS1',
 287                      'PLC_NET_DNS2']:
 288             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 289         fileconf.write('w\n')
 290         fileconf.write('q\n')
 291         fileconf.close()
 292         utils.system('cat %s'%tmpname)
 293         self.test_ssh.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 294         utils.system('rm %s'%tmpname)
 295         return True
 296
 297     # the chroot install is slightly different to this respect
 298     def start(self, options):
 299         if self.vserver:
 300             self.test_ssh.run_in_guest('service plc start')
 301         else:
 302             self.test_ssh.run_in_host('service plc start')
 303         return True
 304
 305     def stop(self, options):
 306         if self.vserver:
 307             self.test_ssh.run_in_guest('service plc stop')
 308         else:
 309             self.test_ssh.run_in_host('service plc stop')
 310         return True
 311
 312     # could use a TestKey class
 313     def store_keys(self, options):
 314         for key_spec in self.plc_spec['keys']:
 315             TestKey(self,key_spec).store_key()
 316         return True
 317
 318     def clean_keys(self, options):
 319         utils.system("rm -rf %s/keys/"%self.path)
 320
 321     def sites (self,options):
 322         return self.do_sites(options)
 323
 324     def clean_sites (self,options):
 325         return self.do_sites(options,action="delete")
 326
 327     def do_sites (self,options,action="add"):
 328         for site_spec in self.plc_spec['sites']:
 329             test_site = TestSite (self,site_spec)
 330             if (action != "add"):
 331                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 332                 test_site.delete_site()
 333                 # deleted with the site
 334                 #test_site.delete_users()
 335                 continue
 336             else:
 337                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 338                 test_site.create_site()
 339                 test_site.create_users()
 340         return True
 341
 342     def nodes (self, options):
 343         return self.do_nodes(options)
 344     def clean_nodes (self, options):
 345         return self.do_nodes(options,action="delete")
 346
 347     def do_nodes (self, options,action="add"):
 348         for site_spec in self.plc_spec['sites']:
 349             test_site = TestSite (self,site_spec)
 350             if action != "add":
 351                 utils.header("Deleting nodes in site %s"%test_site.name())
 352                 for node_spec in site_spec['nodes']:
 353                     test_node=TestNode(self,test_site,node_spec)
 354                     utils.header("Deleting %s"%test_node.name())
 355                     test_node.delete_node()
 356             else:
 357                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 358                 for node_spec in site_spec['nodes']:
 359                     utils.pprint('Creating node %s'%node_spec,node_spec)
 360                     test_node = TestNode (self,test_site,node_spec)
 361                     test_node.create_node ()
 362         return True
 363
 364     # create nodegroups if needed, and populate
 365     # no need for a clean_nodegroups if we are careful enough
 366     def nodegroups (self, options):
 367         # 1st pass to scan contents
 368         groups_dict = {}
 369         for site_spec in self.plc_spec['sites']:
 370             test_site = TestSite (self,site_spec)
 371             for node_spec in site_spec['nodes']:
 372                 test_node=TestNode (self,test_site,node_spec)
 373                 if node_spec.has_key('nodegroups'):
 374                     nodegroupnames=node_spec['nodegroups']
 375                     if isinstance(nodegroupnames,StringTypes):
 376                         nodegroupnames = [ nodegroupnames ]
 377                     for nodegroupname in nodegroupnames:
 378                         if not groups_dict.has_key(nodegroupname):
 379                             groups_dict[nodegroupname]=[]
 380                         groups_dict[nodegroupname].append(test_node.name())
 381         auth=self.auth_root()
 382         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 383             try:
 384                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 385             except:
 386                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 387             for node in group_nodes:
 388                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 389         return True
 390
 391     def all_hostnames (self) :
 392         hostnames = []
 393         for site_spec in self.plc_spec['sites']:
 394             hostnames += [ node_spec['node_fields']['hostname'] \
 395                            for node_spec in site_spec['nodes'] ]
 396         return hostnames
 397
 398     # gracetime : during the first <gracetime> minutes nothing gets printed
 399     def do_nodes_booted (self, minutes, gracetime=2):
 400         # compute timeout
 401         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 402         graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
 403         # the nodes that haven't checked yet - start with a full list and shrink over time
 404         tocheck = self.all_hostnames()
 405         utils.header("checking nodes %r"%tocheck)
 406         # create a dict hostname -> status
 407         status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
 408         while tocheck:
 409             # get their status
 410             tocheck_status=self.server.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
 411             # update status
 412             for array in tocheck_status:
 413                 hostname=array['hostname']
 414                 boot_state=array['boot_state']
 415                 if boot_state == 'boot':
 416                     utils.header ("%s has reached the 'boot' state"%hostname)
 417                 else:
 418                     # if it's a real node, never mind
 419                     (site_spec,node_spec)=self.locate_node(hostname)
 420                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 421                         utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
 422                         # let's cheat
 423                         boot_state = 'boot'
 424                     if datetime.datetime.now() > graceout:
 425                         utils.header ("%s still in '%s' state"%(hostname,boot_state))
 426                         graceout=datetime.datetime.now()+datetime.timedelta(1)
 427                 status[hostname] = boot_state
 428             # refresh tocheck
 429             tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
 430             if not tocheck:
 431                 return True
 432             if datetime.datetime.now() > timeout:
 433                 for hostname in tocheck:
 434                     utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
 435                 return False
 436             # otherwise, sleep for a while
 437             time.sleep(15)
 438         # only useful in empty plcs
 439         return True
 440
 441     def nodes_booted(self,options):
 442         return self.do_nodes_booted(minutes=5)
 443
 444     #to scan and store the nodes's public keys and avoid to ask for confirmation when  ssh
 445     def scan_publicKeys(self,hostnames):
 446         try:
 447             temp_knownhosts="/root/known_hosts"
 448             remote_knownhosts="/root/.ssh/known_hosts"
 449             self.test_ssh.run_in_host("touch %s"%temp_knownhosts )
 450             for hostname in hostnames:
 451                 utils.header("Scan Public %s key and store it in the known_host file(under the root image) "%hostname)
 452                 scan=self.test_ssh.run_in_host('ssh-keyscan -t rsa %s >> %s '%(hostname,temp_knownhosts))
 453             #Store the public keys in the right root image
 454             self.copy_in_guest(temp_knownhosts,remote_knownhosts,True)
 455             #clean the temp keys file used
 456             self.test_ssh.run_in_host('rm -f  %s '%temp_knownhosts )
 457         except Exception, err:
 458             print err
 459
 460     def do_check_nodesSsh(self,minutes):
 461         # compute timeout
 462         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 463         tocheck = self.all_hostnames()
 464         self.scan_publicKeys(tocheck)
 465         utils.header("checking Connectivity on nodes %r"%tocheck)
 466         while tocheck:
 467             for hostname in tocheck:
 468                 # try to ssh in nodes
 469                 access=self.test_ssh.run_in_guest('ssh -i /etc/planetlab/root_ssh_key.rsa root@%s date'%hostname )
 470                 if (not access):
 471                     utils.header('The node %s is sshable -->'%hostname)
 472                     # refresh tocheck
 473                     tocheck.remove(hostname)
 474                 else:
 475                     (site_spec,node_spec)=self.locate_node(hostname)
 476                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 477                         utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
 478                         tocheck.remove(hostname)
 479             if  not tocheck:
 480                 return True
 481             if datetime.datetime.now() > timeout:
 482                 for hostname in tocheck:
 483                     utils.header("FAILURE to ssh into %s"%hostname)
 484                 return False
 485             # otherwise, sleep for a while
 486             time.sleep(15)
 487         # only useful in empty plcs
 488         return True
 489
 490     def nodes_ssh(self, options):
 491         return  self.do_check_nodesSsh(minutes=2)
 492
 493     def bootcd (self, options):
 494         for site_spec in self.plc_spec['sites']:
 495             test_site = TestSite (self,site_spec)
 496             for node_spec in site_spec['nodes']:
 497                 test_node=TestNode (self,test_site,node_spec)
 498                 test_node.prepare_area()
 499                 test_node.create_boot_cd()
 500                 test_node.configure_qemu()
 501         return True
 502
 503     def do_check_intiscripts(self):
 504         for site_spec in self.plc_spec['sites']:
 505                 test_site = TestSite (self,site_spec)
 506                 test_node = TestNode (self,test_site,site_spec['nodes'])
 507                 for slice_spec in self.plc_spec['slices']:
 508                         test_slice=TestSlice (self,test_site,slice_spec)
 509                         test_sliver=TestSliver(self,test_node,test_slice)
 510                         init_status=test_sliver.get_initscript(slice_spec)
 511                         if (not init_status):
 512                                 return False
 513                 return init_status
 514
 515     def check_initscripts(self, options):
 516             return self.do_check_intiscripts()
 517
 518     def initscripts (self, options):
 519         for initscript in self.plc_spec['initscripts']:
 520             utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 521             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 522         return True
 523
 524     def slices (self, options):
 525         return self.do_slices()
 526
 527     def clean_slices (self, options):
 528         return self.do_slices("delete")
 529
 530     def do_slices (self,  action="add"):
 531         for slice in self.plc_spec['slices']:
 532             site_spec = self.locate_site (slice['sitename'])
 533             test_site = TestSite(self,site_spec)
 534             test_slice=TestSlice(self,test_site,slice)
 535             if action != "add":
 536                 utils.header("Deleting slices in site %s"%test_site.name())
 537                 test_slice.delete_slice()
 538             else:
 539                 utils.pprint("Creating slice",slice)
 540                 test_slice.create_slice()
 541                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
 542         return True
 543
 544     def check_slices(self, options):
 545         for slice_spec in self.plc_spec['slices']:
 546             site_spec = self.locate_site (slice_spec['sitename'])
 547             test_site = TestSite(self,site_spec)
 548             test_slice=TestSlice(self,test_site,slice_spec)
 549             status=test_slice.do_check_slice(options)
 550             if (not status):
 551                 return False
 552         return status
 553
 554     def start_nodes (self, options):
 555         utils.header("Starting  nodes")
 556         for site_spec in self.plc_spec['sites']:
 557             TestSite(self,site_spec).start_nodes (options)
 558         return True
 559
 560     def stop_nodes (self, options):
 561         self.kill_all_qemus(options)
 562         return True
 563
 564     def check_tcp (self, options):
 565             #we just need to create a sliver object nothing else
 566             test_sliver=TestSliver(self,
 567                                    TestNode(self, TestSite(self,self.plc_spec['sites'][0]),
 568                                             self.plc_spec['sites'][0]['nodes'][0]),
 569                                    TestSlice(self,TestSite(self,self.plc_spec['sites'][0]),
 570                                              self.plc_spec['slices']))
 571             return test_sliver.do_check_tcp(self.plc_spec['tcp_param'],options)
 572
 573     # returns the filename to use for sql dump/restore, using options.dbname if set
 574     def dbfile (self, database, options):
 575         # uses options.dbname if it is found
 576         try:
 577             name=options.dbname
 578             if not isinstance(name,StringTypes):
 579                 raise Exception
 580         except:
 581             t=datetime.datetime.now()
 582             d=t.date()
 583             name=str(d)
 584         return "/root/%s-%s.sql"%(database,name)
 585
 586     def db_dump(self, options):
 587
 588         dump=self.dbfile("planetab4",options)
 589         self.test_ssh.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 590         utils.header('Dumped planetlab4 database in %s'%dump)
 591         return True
 592
 593     def db_restore(self, options):
 594         dump=self.dbfile("planetab4",options)
 595         ##stop httpd service
 596         self.test_ssh.run_in_guest('service httpd stop')
 597         # xxx - need another wrapper
 598         self.test_ssh.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 599         self.test_ssh.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 600         self.test_ssh.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 601         ##starting httpd service
 602         self.test_ssh.run_in_guest('service httpd start')
 603
 604         utils.header('Database restored from ' + dump)
 605
 606     @standby_generic
 607     def standby_1(): pass
 608     @standby_generic
 609     def standby_2(): pass
 610     @standby_generic
 611     def standby_3(): pass
 612     @standby_generic
 613     def standby_4(): pass
 614     @standby_generic
 615     def standby_5(): pass
 616     @standby_generic
 617     def standby_6(): pass
 618     @standby_generic
 619     def standby_7(): pass
 620     @standby_generic
 621     def standby_8(): pass
 622     @standby_generic
 623     def standby_9(): pass
 624     @standby_generic
 625     def standby_10(): pass
 626     @standby_generic
 627     def standby_11(): pass
 628     @standby_generic
 629     def standby_12(): pass
 630     @standby_generic
 631     def standby_13(): pass
 632     @standby_generic
 633     def standby_14(): pass
 634     @standby_generic
 635     def standby_15(): pass
 636     @standby_generic
 637     def standby_16(): pass
 638     @standby_generic
 639     def standby_17(): pass
 640     @standby_generic
 641     def standby_18(): pass
 642     @standby_generic
 643     def standby_19(): pass
 644     @standby_generic
 645     def standby_20(): pass
 646