system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16 from TestSlice import TestSlice
  17 from TestSliver import TestSliver
  18 from TestBox import TestBox
  19
  20 # step methods must take (self, options) and return a boolean
  21
  22 def standby(minutes):
  23         utils.header('Entering StandBy for %d mn'%minutes)
  24         time.sleep(60*minutes)
  25         return True
  26
  27 def standby_generic (func):
  28     def actual(self,options):
  29         minutes=int(func.__name__.split("_")[1])
  30         return standby(minutes)
  31     return actual
  32
  33 class TestPlc:
  34
  35     def __init__ (self,plc_spec):
  36         self.plc_spec=plc_spec
  37         self.path=os.path.dirname(sys.argv[0])
  38         try:
  39             self.vserverip=plc_spec['vserverip']
  40             self.vservername=plc_spec['vservername']
  41             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  42             self.vserver=True
  43         except:
  44             self.vserver=False
  45             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  46         utils.header('Using API url %s'%self.url)
  47         self.server=xmlrpclib.Server(self.url,allow_none=True)
  48
  49     def name(self):
  50         name=self.plc_spec['name']
  51         if self.vserver:
  52             return name+"[%s]"%self.vservername
  53         else:
  54             return name+"[chroot]"
  55
  56     def is_local (self):
  57         return self.plc_spec['hostname'] == 'localhost'
  58
  59     # define the API methods on this object through xmlrpc
  60     # would help, but not strictly necessary
  61     def connect (self):
  62         pass
  63
  64     # command gets run in the chroot/vserver
  65     def host_to_guest(self,command):
  66         if self.vserver:
  67             return "vserver %s exec %s"%(self.vservername,command)
  68         else:
  69             return "chroot /plc/root %s"%utils.backslash_shell_specials(command)
  70
  71     # command gets run on the right box
  72     def to_host(self,command):
  73         if self.is_local():
  74             return command
  75         else:
  76             return "ssh %s %s"%(self.plc_spec['hostname'],utils.backslash_shell_specials(command))
  77
  78     def full_command(self,command):
  79         return self.to_host(self.host_to_guest(command))
  80
  81     def run_in_guest (self,command):
  82         return utils.system(self.full_command(command))
  83     def run_in_host (self,command):
  84         return utils.system(self.to_host(command))
  85
  86     # xxx quick n dirty
  87     def run_in_guest_piped (self,local,remote):
  88         return utils.system(local+" | "+self.full_command(remote))
  89
  90     # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
  91     def copy_in_guest (self, localfile, remotefile, in_data=False):
  92         if in_data:
  93             chroot_dest="/plc/data"
  94         else:
  95             chroot_dest="/plc/root"
  96         if self.is_local():
  97             if not self.vserver:
  98                 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
  99             else:
 100                 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
 101         else:
 102             if not self.vserver:
 103                 utils.system("scp %s %s:%s/%s"%(localfile,self.plc_spec['hostname'],chroot_dest,remotefile))
 104             else:
 105                 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.plc_spec['hostname'],self.vservername,remotefile))
 106
 107     def auth_root (self):
 108         return {'Username':self.plc_spec['PLC_ROOT_USER'],
 109                 'AuthMethod':'password',
 110                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
 111                 'Role' : self.plc_spec['role']
 112                 }
 113     def locate_site (self,sitename):
 114         for site in self.plc_spec['sites']:
 115             if site['site_fields']['name'] == sitename:
 116                 return site
 117             if site['site_fields']['login_base'] == sitename:
 118                 return site
 119         raise Exception,"Cannot locate site %s"%sitename
 120
 121     def locate_node (self,nodename):
 122         for site in self.plc_spec['sites']:
 123             for node in site['nodes']:
 124                 if node['node_fields']['hostname'] == nodename:
 125                     return (site,node)
 126         raise Exception,"Cannot locate node %s"%nodename
 127
 128     def locate_key (self,keyname):
 129         for key in self.plc_spec['keys']:
 130             if key['name'] == keyname:
 131                 return key
 132         raise Exception,"Cannot locate key %s"%keyname
 133
 134     # all different hostboxes used in this plc
 135     def gather_hostBoxes(self):
 136         # maps on sites and nodes, return [ (host_box,test_node) ]
 137         tuples=[]
 138         for site_spec in self.plc_spec['sites']:
 139             test_site = TestSite (self,site_spec)
 140             for node_spec in site_spec['nodes']:
 141                 test_node = TestNode (self, test_site, node_spec)
 142                 if not test_node.is_real():
 143                     tuples.append( (test_node.host_box(),test_node) )
 144         # transform into a dict { 'host_box' -> [ hostnames .. ] }
 145         result = {}
 146         for (box,node) in tuples:
 147             if not result.has_key(box):
 148                 result[box]=[node]
 149             else:
 150                 result[box].append(node)
 151         return result
 152
 153     # a step for checking this stuff
 154     def showboxes (self,options):
 155         print 'showboxes'
 156         for (box,nodes) in self.gather_hostBoxes().iteritems():
 157             print box,":"," + ".join( [ node.name() for node in nodes ] )
 158         return True
 159
 160     # make this a valid step
 161     def kill_all_qemus(self,options):
 162         for (box,nodes) in self.gather_hostBoxes().iteritems():
 163             # this is the brute force version, kill all qemus on that host box
 164             TestBox(box).kill_all_qemus()
 165         return True
 166
 167     # make this a valid step
 168     def list_all_qemus(self,options):
 169         for (box,nodes) in self.gather_hostBoxes().iteritems():
 170             # push the script
 171             TestBox(box).copy("qemu_kill.sh")
 172             # this is the brute force version, kill all qemus on that host box
 173             TestBox(box).run("./qemu_kill.sh -l")
 174         return True
 175
 176     # kill only the right qemus
 177     def kill_qemus(self,options):
 178         for (box,nodes) in self.gather_hostBoxes().iteritems():
 179             # push the script
 180             TestBox(box).copy("qemu_kill.sh")
 181             # the fine-grain version
 182             for node in nodes:
 183                 node.kill_qemu()
 184         return True
 185
 186     def clear_ssh_config (self,options):
 187         # install local ssh_config file as root's .ssh/config - ssh should be quiet
 188         # dir might need creation first
 189         self.run_in_guest("mkdir /root/.ssh")
 190         self.run_in_guest("chmod 700 /root/.ssh")
 191         # this does not work - > redirection somehow makes it until an argument to cat
 192         #self.run_in_guest_piped("cat ssh_config","cat > /root/.ssh/config")
 193         self.copy_in_guest("ssh_config","/root/.ssh/config",True)
 194         return True
 195
 196     #################### step methods
 197
 198     ### uninstall
 199     def uninstall_chroot(self,options):
 200         self.run_in_host('service plc safestop')
 201         #####detecting the last myplc version installed and remove it
 202         self.run_in_host('rpm -e myplc')
 203         ##### Clean up the /plc directory
 204         self.run_in_host('rm -rf  /plc/data')
 205         ##### stop any running vservers
 206         self.run_in_host('for vserver in $(ls /vservers/* | sed -e s,/vservers/,,) ; do vserver $vserver stop ; done')
 207         return True
 208
 209     def uninstall_vserver(self,options):
 210         self.run_in_host("vserver --silent %s delete"%self.vservername)
 211         return True
 212
 213     def uninstall(self,options):
 214         # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
 215         # it sounds safer to have the former uninstalled too
 216         # now the vserver method cannot be invoked for chroot instances as vservername is required
 217         if self.vserver:
 218             self.uninstall_vserver(options)
 219             self.uninstall_chroot(options)
 220         else:
 221             self.uninstall_chroot(options)
 222         return True
 223
 224     ### install
 225     def install_chroot(self,options):
 226         # nothing to do
 227         return True
 228
 229     # xxx this would not work with hostname != localhost as mylc-init-vserver was extracted locally
 230     def install_vserver(self,options):
 231         # we need build dir for vtest-init-vserver
 232         if self.is_local():
 233             # a full path for the local calls
 234             build_dir=self.path+"/build"
 235         else:
 236             # use a standard name - will be relative to HOME
 237             build_dir="tests-system-build"
 238         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 239         if self.run_in_host(build_checkout) != 0:
 240             raise Exception,"Cannot checkout build dir"
 241         # the repo url is taken from myplc-url
 242         # with the last two steps (i386/myplc...) removed
 243         repo_url = options.myplc_url
 244         repo_url = os.path.dirname(repo_url)
 245         repo_url = os.path.dirname(repo_url)
 246         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 247             (build_dir,self.vservername,repo_url,self.vserverip)
 248         if self.run_in_host(create_vserver) != 0:
 249             raise Exception,"Could not create vserver for %s"%self.vservername
 250         return True
 251
 252     def install(self,options):
 253         if self.vserver:
 254             return self.install_vserver(options)
 255         else:
 256             return self.install_chroot(options)
 257
 258     ### install_rpm
 259     def install_rpm_chroot(self,options):
 260         utils.header('Installing from %s'%options.myplc_url)
 261         url=options.myplc_url
 262         self.run_in_host('rpm -Uvh '+url)
 263         self.run_in_host('service plc mount')
 264         return True
 265
 266     def install_rpm_vserver(self,options):
 267         self.run_in_guest("yum -y install myplc-native")
 268         return True
 269
 270     def install_rpm(self,options):
 271         if self.vserver:
 272             return self.install_rpm_vserver(options)
 273         else:
 274             return self.install_rpm_chroot(options)
 275
 276     ###
 277     def configure(self,options):
 278         tmpname='%s/%s.plc-config-tty'%(options.path,self.name())
 279         fileconf=open(tmpname,'w')
 280         for var in [ 'PLC_NAME',
 281                      'PLC_ROOT_PASSWORD',
 282                      'PLC_ROOT_USER',
 283                      'PLC_MAIL_ENABLED',
 284                      'PLC_MAIL_SUPPORT_ADDRESS',
 285                      'PLC_DB_HOST',
 286                      'PLC_API_HOST',
 287                      'PLC_WWW_HOST',
 288                      'PLC_BOOT_HOST',
 289                      'PLC_NET_DNS1',
 290                      'PLC_NET_DNS2']:
 291             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 292         fileconf.write('w\n')
 293         fileconf.write('q\n')
 294         fileconf.close()
 295         utils.system('cat %s'%tmpname)
 296         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 297         utils.system('rm %s'%tmpname)
 298         return True
 299
 300     # the chroot install is slightly different to this respect
 301     def start(self, options):
 302         if self.vserver:
 303             self.run_in_guest('service plc start')
 304         else:
 305             self.run_in_host('service plc start')
 306         return True
 307
 308     def stop(self, options):
 309         if self.vserver:
 310             self.run_in_guest('service plc stop')
 311         else:
 312             self.run_in_host('service plc stop')
 313         return True
 314
 315     # could use a TestKey class
 316     def store_keys(self, options):
 317         for key_spec in self.plc_spec['keys']:
 318             TestKey(self,key_spec).store_key()
 319         return True
 320
 321     def clean_keys(self, options):
 322         utils.system("rm -rf %s/keys/"%self.path)
 323
 324     def sites (self,options):
 325         return self.do_sites(options)
 326
 327     def clean_sites (self,options):
 328         return self.do_sites(options,action="delete")
 329
 330     def do_sites (self,options,action="add"):
 331         for site_spec in self.plc_spec['sites']:
 332             test_site = TestSite (self,site_spec)
 333             if (action != "add"):
 334                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 335                 test_site.delete_site()
 336                 # deleted with the site
 337                 #test_site.delete_users()
 338                 continue
 339             else:
 340                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 341                 test_site.create_site()
 342                 test_site.create_users()
 343         return True
 344
 345     def nodes (self, options):
 346         return self.do_nodes(options)
 347     def clean_nodes (self, options):
 348         return self.do_nodes(options,action="delete")
 349
 350     def do_nodes (self, options,action="add"):
 351         for site_spec in self.plc_spec['sites']:
 352             test_site = TestSite (self,site_spec)
 353             if action != "add":
 354                 utils.header("Deleting nodes in site %s"%test_site.name())
 355                 for node_spec in site_spec['nodes']:
 356                     test_node=TestNode(self,test_site,node_spec)
 357                     utils.header("Deleting %s"%test_node.name())
 358                     test_node.delete_node()
 359             else:
 360                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 361                 for node_spec in site_spec['nodes']:
 362                     utils.show_spec('Creating node %s'%node_spec,node_spec)
 363                     test_node = TestNode (self,test_site,node_spec)
 364                     test_node.create_node ()
 365         return True
 366
 367     # create nodegroups if needed, and populate
 368     # no need for a clean_nodegroups if we are careful enough
 369     def nodegroups (self, options):
 370         # 1st pass to scan contents
 371         groups_dict = {}
 372         for site_spec in self.plc_spec['sites']:
 373             test_site = TestSite (self,site_spec)
 374             for node_spec in site_spec['nodes']:
 375                 test_node=TestNode (self,test_site,node_spec)
 376                 if node_spec.has_key('nodegroups'):
 377                     nodegroupnames=node_spec['nodegroups']
 378                     if isinstance(nodegroupnames,StringTypes):
 379                         nodegroupnames = [ nodegroupnames ]
 380                     for nodegroupname in nodegroupnames:
 381                         if not groups_dict.has_key(nodegroupname):
 382                             groups_dict[nodegroupname]=[]
 383                         groups_dict[nodegroupname].append(test_node.name())
 384         auth=self.auth_root()
 385         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 386             try:
 387                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 388             except:
 389                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 390             for node in group_nodes:
 391                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 392         return True
 393
 394     def all_hostnames (self) :
 395         hostnames = []
 396         for site_spec in self.plc_spec['sites']:
 397             hostnames += [ node_spec['node_fields']['hostname'] \
 398                            for node_spec in site_spec['nodes'] ]
 399         return hostnames
 400
 401     # gracetime : during the first <gracetime> minutes nothing gets printed
 402     def do_nodes_booted (self, minutes, gracetime=2):
 403         # compute timeout
 404         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 405         graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
 406         # the nodes that haven't checked yet - start with a full list and shrink over time
 407         tocheck = self.all_hostnames()
 408         utils.header("checking nodes %r"%tocheck)
 409         # create a dict hostname -> status
 410         status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
 411         while tocheck:
 412             # get their status
 413             tocheck_status=self.server.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
 414             # update status
 415             for array in tocheck_status:
 416                 hostname=array['hostname']
 417                 boot_state=array['boot_state']
 418                 if boot_state == 'boot':
 419                     utils.header ("%s has reached the 'boot' state"%hostname)
 420                 else:
 421                     # if it's a real node, never mind
 422                     (site_spec,node_spec)=self.locate_node(hostname)
 423                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 424                         utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
 425                         # let's cheat
 426                         boot_state = 'boot'
 427                     if datetime.datetime.now() > graceout:
 428                         utils.header ("%s still in '%s' state"%(hostname,boot_state))
 429                         graceout=datetime.datetime.now()+datetime.timedelta(1)
 430                 status[hostname] = boot_state
 431             # refresh tocheck
 432             tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
 433             if not tocheck:
 434                 return True
 435             if datetime.datetime.now() > timeout:
 436                 for hostname in tocheck:
 437                     utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
 438                 return False
 439             # otherwise, sleep for a while
 440             time.sleep(15)
 441         # only useful in empty plcs
 442         return True
 443
 444     def nodes_booted(self,options):
 445         return self.do_nodes_booted(minutes=5)
 446
 447     #to scan and store the nodes's public keys and avoid to ask for confirmation when  ssh
 448     def scan_publicKeys(self,hostnames):
 449         try:
 450             temp_knownhosts="/root/known_hosts"
 451             remote_knownhosts="/root/.ssh/known_hosts"
 452             self.run_in_host("touch %s"%temp_knownhosts )
 453             for hostname in hostnames:
 454                 utils.header("Scan Public %s key and store it in the known_host file(under the root image) "%hostname)
 455                 scan=self.run_in_host('ssh-keyscan -t rsa %s >> %s '%(hostname,temp_knownhosts))
 456             #Store the public keys in the right root image
 457             self.copy_in_guest(temp_knownhosts,remote_knownhosts,True)
 458             #clean the temp keys file used
 459             self.run_in_host('rm -f  %s '%temp_knownhosts )
 460         except Exception, err:
 461             print err
 462
 463     def do_check_nodesSsh(self,minutes):
 464         # compute timeout
 465         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 466         tocheck = self.all_hostnames()
 467         self.scan_publicKeys(tocheck)
 468         utils.header("checking Connectivity on nodes %r"%tocheck)
 469         while tocheck:
 470             for hostname in tocheck:
 471                 # try to ssh in nodes
 472                 access=self.run_in_guest('ssh -i /etc/planetlab/root_ssh_key.rsa root@%s date'%hostname )
 473                 if (not access):
 474                     utils.header('The node %s is sshable -->'%hostname)
 475                     # refresh tocheck
 476                     tocheck.remove(hostname)
 477                 else:
 478                     (site_spec,node_spec)=self.locate_node(hostname)
 479                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 480                         utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
 481                         tocheck.remove(hostname)
 482             if  not tocheck:
 483                 return True
 484             if datetime.datetime.now() > timeout:
 485                 for hostname in tocheck:
 486                     utils.header("FAILURE to ssh into %s"%hostname)
 487                 return False
 488             # otherwise, sleep for a while
 489             time.sleep(15)
 490         # only useful in empty plcs
 491         return True
 492
 493     def nodes_ssh(self, options):
 494         return  self.do_check_nodesSsh(minutes=2)
 495
 496     def bootcd (self, options):
 497         for site_spec in self.plc_spec['sites']:
 498             test_site = TestSite (self,site_spec)
 499             for node_spec in site_spec['nodes']:
 500                 test_node=TestNode (self,test_site,node_spec)
 501                 test_node.create_boot_cd(options.path)
 502         return True
 503
 504     def do_check_intiscripts(self):
 505         for site_spec in self.plc_spec['sites']:
 506                 test_site = TestSite (self,site_spec)
 507                 test_node = TestNode (self,test_site,site_spec['nodes'])
 508                 for slice_spec in self.plc_spec['slices']:
 509                         test_slice=TestSlice (self,test_site,slice_spec)
 510                         test_sliver=TestSliver(self,test_node,test_slice)
 511                         init_status=test_sliver.get_initscript(slice_spec)
 512                         if (not init_status):
 513                                 return False
 514                 return init_status
 515
 516     def check_initscripts(self, options):
 517             return self.do_check_intiscripts()
 518
 519     def initscripts (self, options):
 520         for initscript in self.plc_spec['initscripts']:
 521             utils.show_spec('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 522             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 523         return True
 524
 525     def slices (self, options):
 526         return self.do_slices()
 527
 528     def clean_slices (self, options):
 529         return self.do_slices("delete")
 530
 531     def do_slices (self,  action="add"):
 532         for slice in self.plc_spec['slices']:
 533             site_spec = self.locate_site (slice['sitename'])
 534             test_site = TestSite(self,site_spec)
 535             test_slice=TestSlice(self,test_site,slice)
 536             if action != "add":
 537                 utils.header("Deleting slices in site %s"%test_site.name())
 538                 test_slice.delete_slice()
 539             else:
 540                 utils.show_spec("Creating slice",slice)
 541                 test_slice.create_slice()
 542                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
 543         return True
 544
 545     def check_slices(self, options):
 546         for slice_spec in self.plc_spec['slices']:
 547             site_spec = self.locate_site (slice_spec['sitename'])
 548             test_site = TestSite(self,site_spec)
 549             test_slice=TestSlice(self,test_site,slice_spec)
 550             status=test_slice.do_check_slice(options)
 551             if (not status):
 552                 return False
 553         return status
 554
 555     def start_nodes (self, options):
 556         utils.header("Starting  nodes")
 557         for site_spec in self.plc_spec['sites']:
 558             TestSite(self,site_spec).start_nodes (options)
 559         return True
 560
 561     def stop_nodes (self, options):
 562         self.kill_all_qemus()
 563         return True
 564
 565     def check_tcp (self, options):
 566             #we just need to create a sliver object nothing else
 567             test_sliver=TestSliver(self,
 568                                    TestNode(self, TestSite(self,self.plc_spec['sites'][0]),
 569                                             self.plc_spec['sites'][0]['nodes'][0]),
 570                                    TestSlice(self,TestSite(self,self.plc_spec['sites'][0]),
 571                                              self.plc_spec['slices']))
 572             return test_sliver.do_check_tcp(self.plc_spec['tcp_param'],options)
 573
 574     # returns the filename to use for sql dump/restore, using options.dbname if set
 575     def dbfile (self, database, options):
 576         # uses options.dbname if it is found
 577         try:
 578             name=options.dbname
 579             if not isinstance(name,StringTypes):
 580                 raise Exception
 581         except:
 582             t=datetime.datetime.now()
 583             d=t.date()
 584             name=str(d)
 585         return "/root/%s-%s.sql"%(database,name)
 586
 587     def db_dump(self, options):
 588
 589         dump=self.dbfile("planetab4",options)
 590         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 591         utils.header('Dumped planetlab4 database in %s'%dump)
 592         return True
 593
 594     def db_restore(self, options):
 595         dump=self.dbfile("planetab4",options)
 596         ##stop httpd service
 597         self.run_in_guest('service httpd stop')
 598         # xxx - need another wrapper
 599         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 600         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 601         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 602         ##starting httpd service
 603         self.run_in_guest('service httpd start')
 604
 605         utils.header('Database restored from ' + dump)
 606
 607     @standby_generic
 608     def standby_1(): pass
 609     @standby_generic
 610     def standby_2(): pass
 611     @standby_generic
 612     def standby_3(): pass
 613     @standby_generic
 614     def standby_4(): pass
 615     @standby_generic
 616     def standby_5(): pass
 617     @standby_generic
 618     def standby_6(): pass
 619     @standby_generic
 620     def standby_7(): pass
 621     @standby_generic
 622     def standby_8(): pass
 623     @standby_generic
 624     def standby_9(): pass
 625     @standby_generic
 626     def standby_10(): pass
 627     @standby_generic
 628     def standby_11(): pass
 629     @standby_generic
 630     def standby_12(): pass
 631     @standby_generic
 632     def standby_13(): pass
 633     @standby_generic
 634     def standby_14(): pass
 635     @standby_generic
 636     def standby_15(): pass
 637     @standby_generic
 638     def standby_16(): pass
 639     @standby_generic
 640     def standby_17(): pass
 641     @standby_generic
 642     def standby_18(): pass
 643     @standby_generic
 644     def standby_19(): pass
 645     @standby_generic
 646     def standby_20(): pass
 647