system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16 from TestSlice import TestSlice
  17 from TestSliver import TestSliver
  18 from TestBox import TestBox
  19
  20 # step methods must take (self, options) and return a boolean
  21
  22 def standby(minutes):
  23         utils.header('Entering StandBy for %d mn'%minutes)
  24         time.sleep(60*minutes)
  25         return True
  26
  27 def standby_generic (func):
  28     def actual(self,options):
  29         minutes=int(func.__name__.split("_")[1])
  30         return standby(minutes)
  31     return actual
  32
  33 class TestPlc:
  34
  35     def __init__ (self,plc_spec):
  36         self.plc_spec=plc_spec
  37         self.path=os.path.dirname(sys.argv[0])
  38         try:
  39             self.vserverip=plc_spec['vserverip']
  40             self.vservername=plc_spec['vservername']
  41             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  42             self.vserver=True
  43         except:
  44             self.vserver=False
  45             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  46         utils.header('Using API url %s'%self.url)
  47         self.server=xmlrpclib.Server(self.url,allow_none=True)
  48
  49     def name(self):
  50         name=self.plc_spec['name']
  51         if self.vserver:
  52             return name+"[%s]"%self.vservername
  53         else:
  54             return name+"[chroot]"
  55
  56     def hostname(self):
  57         return self.plc_spec['hostname']
  58
  59     def is_local (self):
  60         return utils.is_local(self.hostname())
  61
  62     # define the API methods on this object through xmlrpc
  63     # would help, but not strictly necessary
  64     def connect (self):
  65         pass
  66
  67     # command gets run in the chroot/vserver
  68     def host_to_guest(self,command):
  69         if self.vserver:
  70             return "vserver %s exec %s"%(self.vservername,command)
  71         else:
  72             return "chroot /plc/root %s"%utils.backslash_shell_specials(command)
  73
  74     # command gets run on the right box
  75     def to_host(self,command):
  76         if self.is_local():
  77             return command
  78         else:
  79             return "ssh %s %s"%(self.hostname(),utils.backslash_shell_specials(command))
  80
  81     def full_command(self,command):
  82         return self.to_host(self.host_to_guest(command))
  83
  84     def run_in_guest (self,command):
  85         return utils.system(self.full_command(command))
  86     def run_in_host (self,command):
  87         return utils.system(self.to_host(command))
  88
  89     # xxx quick n dirty
  90     def run_in_guest_piped (self,local,remote):
  91         return utils.system(local+" | "+self.full_command(remote))
  92
  93     # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
  94     def copy_in_guest (self, localfile, remotefile, in_data=False):
  95         if in_data:
  96             chroot_dest="/plc/data"
  97         else:
  98             chroot_dest="/plc/root"
  99         if self.is_local():
 100             if not self.vserver:
 101                 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
 102             else:
 103                 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
 104         else:
 105             if not self.vserver:
 106                 utils.system("scp %s %s:%s/%s"%(localfile,self.hostname(),chroot_dest,remotefile))
 107             else:
 108                 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.hostname(),self.vservername,remotefile))
 109
 110     def auth_root (self):
 111         return {'Username':self.plc_spec['PLC_ROOT_USER'],
 112                 'AuthMethod':'password',
 113                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
 114                 'Role' : self.plc_spec['role']
 115                 }
 116     def locate_site (self,sitename):
 117         for site in self.plc_spec['sites']:
 118             if site['site_fields']['name'] == sitename:
 119                 return site
 120             if site['site_fields']['login_base'] == sitename:
 121                 return site
 122         raise Exception,"Cannot locate site %s"%sitename
 123
 124     def locate_node (self,nodename):
 125         for site in self.plc_spec['sites']:
 126             for node in site['nodes']:
 127                 if node['node_fields']['hostname'] == nodename:
 128                     return (site,node)
 129         raise Exception,"Cannot locate node %s"%nodename
 130
 131     def locate_key (self,keyname):
 132         for key in self.plc_spec['keys']:
 133             if key['name'] == keyname:
 134                 return key
 135         raise Exception,"Cannot locate key %s"%keyname
 136
 137     # all different hostboxes used in this plc
 138     def gather_hostBoxes(self):
 139         # maps on sites and nodes, return [ (host_box,test_node) ]
 140         tuples=[]
 141         for site_spec in self.plc_spec['sites']:
 142             test_site = TestSite (self,site_spec)
 143             for node_spec in site_spec['nodes']:
 144                 test_node = TestNode (self, test_site, node_spec)
 145                 if not test_node.is_real():
 146                     tuples.append( (test_node.host_box(),test_node) )
 147         # transform into a dict { 'host_box' -> [ hostnames .. ] }
 148         result = {}
 149         for (box,node) in tuples:
 150             if not result.has_key(box):
 151                 result[box]=[node]
 152             else:
 153                 result[box].append(node)
 154         return result
 155
 156     # a step for checking this stuff
 157     def showboxes (self,options):
 158         print 'showboxes'
 159         for (box,nodes) in self.gather_hostBoxes().iteritems():
 160             print box,":"," + ".join( [ node.name() for node in nodes ] )
 161         return True
 162
 163     # make this a valid step
 164     def kill_all_qemus(self,options):
 165         for (box,nodes) in self.gather_hostBoxes().iteritems():
 166             # this is the brute force version, kill all qemus on that host box
 167             TestBox(box).kill_all_qemus()
 168         return True
 169
 170     # make this a valid step
 171     def list_all_qemus(self,options):
 172         for (box,nodes) in self.gather_hostBoxes().iteritems():
 173             # push the script
 174             TestBox(box).copy("qemu_kill.sh")
 175             # this is the brute force version, kill all qemus on that host box
 176             TestBox(box).run("./qemu_kill.sh -l")
 177         return True
 178
 179     # kill only the right qemus
 180     def kill_qemus(self,options):
 181         for (box,nodes) in self.gather_hostBoxes().iteritems():
 182             # push the script
 183             TestBox(box).copy("qemu_kill.sh")
 184             # the fine-grain version
 185             for node in nodes:
 186                 node.kill_qemu()
 187         return True
 188
 189     def clear_ssh_config (self,options):
 190         # install local ssh_config file as root's .ssh/config - ssh should be quiet
 191         # dir might need creation first
 192         self.run_in_guest("mkdir /root/.ssh")
 193         self.run_in_guest("chmod 700 /root/.ssh")
 194         # this does not work - > redirection somehow makes it until an argument to cat
 195         #self.run_in_guest_piped("cat ssh_config","cat > /root/.ssh/config")
 196         self.copy_in_guest("ssh_config","/root/.ssh/config",True)
 197         return True
 198
 199     #################### step methods
 200
 201     ### uninstall
 202     def uninstall_chroot(self,options):
 203         self.run_in_host('service plc safestop')
 204         #####detecting the last myplc version installed and remove it
 205         self.run_in_host('rpm -e myplc')
 206         ##### Clean up the /plc directory
 207         self.run_in_host('rm -rf  /plc/data')
 208         ##### stop any running vservers
 209         self.run_in_host('for vserver in $(ls /vservers/* | sed -e s,/vservers/,,) ; do vserver $vserver stop ; done')
 210         return True
 211
 212     def uninstall_vserver(self,options):
 213         self.run_in_host("vserver --silent %s delete"%self.vservername)
 214         return True
 215
 216     def uninstall(self,options):
 217         # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
 218         # it sounds safer to have the former uninstalled too
 219         # now the vserver method cannot be invoked for chroot instances as vservername is required
 220         if self.vserver:
 221             self.uninstall_vserver(options)
 222             self.uninstall_chroot(options)
 223         else:
 224             self.uninstall_chroot(options)
 225         return True
 226
 227     ### install
 228     def install_chroot(self,options):
 229         # nothing to do
 230         return True
 231
 232     # xxx this would not work with hostname != localhost as mylc-init-vserver was extracted locally
 233     def install_vserver(self,options):
 234         # we need build dir for vtest-init-vserver
 235         if self.is_local():
 236             # a full path for the local calls
 237             build_dir=self.path+"/build"
 238         else:
 239             # use a standard name - will be relative to HOME
 240             build_dir="tests-system-build"
 241         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 242         if self.run_in_host(build_checkout) != 0:
 243             raise Exception,"Cannot checkout build dir"
 244         # the repo url is taken from myplc-url
 245         # with the last two steps (i386/myplc...) removed
 246         repo_url = options.myplc_url
 247         repo_url = os.path.dirname(repo_url)
 248         repo_url = os.path.dirname(repo_url)
 249         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 250             (build_dir,self.vservername,repo_url,self.vserverip)
 251         if self.run_in_host(create_vserver) != 0:
 252             raise Exception,"Could not create vserver for %s"%self.vservername
 253         return True
 254
 255     def install(self,options):
 256         if self.vserver:
 257             return self.install_vserver(options)
 258         else:
 259             return self.install_chroot(options)
 260
 261     ### install_rpm
 262     def install_rpm_chroot(self,options):
 263         utils.header('Installing from %s'%options.myplc_url)
 264         url=options.myplc_url
 265         self.run_in_host('rpm -Uvh '+url)
 266         self.run_in_host('service plc mount')
 267         return True
 268
 269     def install_rpm_vserver(self,options):
 270         self.run_in_guest("yum -y install myplc-native")
 271         return True
 272
 273     def install_rpm(self,options):
 274         if self.vserver:
 275             return self.install_rpm_vserver(options)
 276         else:
 277             return self.install_rpm_chroot(options)
 278
 279     ###
 280     def configure(self,options):
 281         tmpname='%s/%s.plc-config-tty'%(options.path,self.name())
 282         fileconf=open(tmpname,'w')
 283         for var in [ 'PLC_NAME',
 284                      'PLC_ROOT_PASSWORD',
 285                      'PLC_ROOT_USER',
 286                      'PLC_MAIL_ENABLED',
 287                      'PLC_MAIL_SUPPORT_ADDRESS',
 288                      'PLC_DB_HOST',
 289                      'PLC_API_HOST',
 290                      'PLC_WWW_HOST',
 291                      'PLC_BOOT_HOST',
 292                      'PLC_NET_DNS1',
 293                      'PLC_NET_DNS2']:
 294             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 295         fileconf.write('w\n')
 296         fileconf.write('q\n')
 297         fileconf.close()
 298         utils.system('cat %s'%tmpname)
 299         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 300         utils.system('rm %s'%tmpname)
 301         return True
 302
 303     # the chroot install is slightly different to this respect
 304     def start(self, options):
 305         if self.vserver:
 306             self.run_in_guest('service plc start')
 307         else:
 308             self.run_in_host('service plc start')
 309         return True
 310
 311     def stop(self, options):
 312         if self.vserver:
 313             self.run_in_guest('service plc stop')
 314         else:
 315             self.run_in_host('service plc stop')
 316         return True
 317
 318     # could use a TestKey class
 319     def store_keys(self, options):
 320         for key_spec in self.plc_spec['keys']:
 321             TestKey(self,key_spec).store_key()
 322         return True
 323
 324     def clean_keys(self, options):
 325         utils.system("rm -rf %s/keys/"%self.path)
 326
 327     def sites (self,options):
 328         return self.do_sites(options)
 329
 330     def clean_sites (self,options):
 331         return self.do_sites(options,action="delete")
 332
 333     def do_sites (self,options,action="add"):
 334         for site_spec in self.plc_spec['sites']:
 335             test_site = TestSite (self,site_spec)
 336             if (action != "add"):
 337                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 338                 test_site.delete_site()
 339                 # deleted with the site
 340                 #test_site.delete_users()
 341                 continue
 342             else:
 343                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 344                 test_site.create_site()
 345                 test_site.create_users()
 346         return True
 347
 348     def nodes (self, options):
 349         return self.do_nodes(options)
 350     def clean_nodes (self, options):
 351         return self.do_nodes(options,action="delete")
 352
 353     def do_nodes (self, options,action="add"):
 354         for site_spec in self.plc_spec['sites']:
 355             test_site = TestSite (self,site_spec)
 356             if action != "add":
 357                 utils.header("Deleting nodes in site %s"%test_site.name())
 358                 for node_spec in site_spec['nodes']:
 359                     test_node=TestNode(self,test_site,node_spec)
 360                     utils.header("Deleting %s"%test_node.name())
 361                     test_node.delete_node()
 362             else:
 363                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 364                 for node_spec in site_spec['nodes']:
 365                     utils.pprint('Creating node %s'%node_spec,node_spec)
 366                     test_node = TestNode (self,test_site,node_spec)
 367                     test_node.create_node ()
 368         return True
 369
 370     # create nodegroups if needed, and populate
 371     # no need for a clean_nodegroups if we are careful enough
 372     def nodegroups (self, options):
 373         # 1st pass to scan contents
 374         groups_dict = {}
 375         for site_spec in self.plc_spec['sites']:
 376             test_site = TestSite (self,site_spec)
 377             for node_spec in site_spec['nodes']:
 378                 test_node=TestNode (self,test_site,node_spec)
 379                 if node_spec.has_key('nodegroups'):
 380                     nodegroupnames=node_spec['nodegroups']
 381                     if isinstance(nodegroupnames,StringTypes):
 382                         nodegroupnames = [ nodegroupnames ]
 383                     for nodegroupname in nodegroupnames:
 384                         if not groups_dict.has_key(nodegroupname):
 385                             groups_dict[nodegroupname]=[]
 386                         groups_dict[nodegroupname].append(test_node.name())
 387         auth=self.auth_root()
 388         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 389             try:
 390                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 391             except:
 392                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 393             for node in group_nodes:
 394                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 395         return True
 396
 397     def all_hostnames (self) :
 398         hostnames = []
 399         for site_spec in self.plc_spec['sites']:
 400             hostnames += [ node_spec['node_fields']['hostname'] \
 401                            for node_spec in site_spec['nodes'] ]
 402         return hostnames
 403
 404     # gracetime : during the first <gracetime> minutes nothing gets printed
 405     def do_nodes_booted (self, minutes, gracetime=2):
 406         # compute timeout
 407         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 408         graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
 409         # the nodes that haven't checked yet - start with a full list and shrink over time
 410         tocheck = self.all_hostnames()
 411         utils.header("checking nodes %r"%tocheck)
 412         # create a dict hostname -> status
 413         status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
 414         while tocheck:
 415             # get their status
 416             tocheck_status=self.server.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
 417             # update status
 418             for array in tocheck_status:
 419                 hostname=array['hostname']
 420                 boot_state=array['boot_state']
 421                 if boot_state == 'boot':
 422                     utils.header ("%s has reached the 'boot' state"%hostname)
 423                 else:
 424                     # if it's a real node, never mind
 425                     (site_spec,node_spec)=self.locate_node(hostname)
 426                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 427                         utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
 428                         # let's cheat
 429                         boot_state = 'boot'
 430                     if datetime.datetime.now() > graceout:
 431                         utils.header ("%s still in '%s' state"%(hostname,boot_state))
 432                         graceout=datetime.datetime.now()+datetime.timedelta(1)
 433                 status[hostname] = boot_state
 434             # refresh tocheck
 435             tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
 436             if not tocheck:
 437                 return True
 438             if datetime.datetime.now() > timeout:
 439                 for hostname in tocheck:
 440                     utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
 441                 return False
 442             # otherwise, sleep for a while
 443             time.sleep(15)
 444         # only useful in empty plcs
 445         return True
 446
 447     def nodes_booted(self,options):
 448         return self.do_nodes_booted(minutes=5)
 449
 450     #to scan and store the nodes's public keys and avoid to ask for confirmation when  ssh
 451     def scan_publicKeys(self,hostnames):
 452         try:
 453             temp_knownhosts="/root/known_hosts"
 454             remote_knownhosts="/root/.ssh/known_hosts"
 455             self.run_in_host("touch %s"%temp_knownhosts )
 456             for hostname in hostnames:
 457                 utils.header("Scan Public %s key and store it in the known_host file(under the root image) "%hostname)
 458                 scan=self.run_in_host('ssh-keyscan -t rsa %s >> %s '%(hostname,temp_knownhosts))
 459             #Store the public keys in the right root image
 460             self.copy_in_guest(temp_knownhosts,remote_knownhosts,True)
 461             #clean the temp keys file used
 462             self.run_in_host('rm -f  %s '%temp_knownhosts )
 463         except Exception, err:
 464             print err
 465
 466     def do_check_nodesSsh(self,minutes):
 467         # compute timeout
 468         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 469         tocheck = self.all_hostnames()
 470         self.scan_publicKeys(tocheck)
 471         utils.header("checking Connectivity on nodes %r"%tocheck)
 472         while tocheck:
 473             for hostname in tocheck:
 474                 # try to ssh in nodes
 475                 access=self.run_in_guest('ssh -i /etc/planetlab/root_ssh_key.rsa root@%s date'%hostname )
 476                 if (not access):
 477                     utils.header('The node %s is sshable -->'%hostname)
 478                     # refresh tocheck
 479                     tocheck.remove(hostname)
 480                 else:
 481                     (site_spec,node_spec)=self.locate_node(hostname)
 482                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 483                         utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
 484                         tocheck.remove(hostname)
 485             if  not tocheck:
 486                 return True
 487             if datetime.datetime.now() > timeout:
 488                 for hostname in tocheck:
 489                     utils.header("FAILURE to ssh into %s"%hostname)
 490                 return False
 491             # otherwise, sleep for a while
 492             time.sleep(15)
 493         # only useful in empty plcs
 494         return True
 495
 496     def nodes_ssh(self, options):
 497         return  self.do_check_nodesSsh(minutes=2)
 498
 499     def bootcd (self, options):
 500         for site_spec in self.plc_spec['sites']:
 501             test_site = TestSite (self,site_spec)
 502             for node_spec in site_spec['nodes']:
 503                 test_node=TestNode (self,test_site,node_spec)
 504                 test_node.create_boot_cd(options.path)
 505         return True
 506
 507     def do_check_intiscripts(self):
 508         for site_spec in self.plc_spec['sites']:
 509                 test_site = TestSite (self,site_spec)
 510                 test_node = TestNode (self,test_site,site_spec['nodes'])
 511                 for slice_spec in self.plc_spec['slices']:
 512                         test_slice=TestSlice (self,test_site,slice_spec)
 513                         test_sliver=TestSliver(self,test_node,test_slice)
 514                         init_status=test_sliver.get_initscript(slice_spec)
 515                         if (not init_status):
 516                                 return False
 517                 return init_status
 518
 519     def check_initscripts(self, options):
 520             return self.do_check_intiscripts()
 521
 522     def initscripts (self, options):
 523         for initscript in self.plc_spec['initscripts']:
 524             utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 525             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 526         return True
 527
 528     def slices (self, options):
 529         return self.do_slices()
 530
 531     def clean_slices (self, options):
 532         return self.do_slices("delete")
 533
 534     def do_slices (self,  action="add"):
 535         for slice in self.plc_spec['slices']:
 536             site_spec = self.locate_site (slice['sitename'])
 537             test_site = TestSite(self,site_spec)
 538             test_slice=TestSlice(self,test_site,slice)
 539             if action != "add":
 540                 utils.header("Deleting slices in site %s"%test_site.name())
 541                 test_slice.delete_slice()
 542             else:
 543                 utils.pprint("Creating slice",slice)
 544                 test_slice.create_slice()
 545                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
 546         return True
 547
 548     def check_slices(self, options):
 549         for slice_spec in self.plc_spec['slices']:
 550             site_spec = self.locate_site (slice_spec['sitename'])
 551             test_site = TestSite(self,site_spec)
 552             test_slice=TestSlice(self,test_site,slice_spec)
 553             status=test_slice.do_check_slice(options)
 554             if (not status):
 555                 return False
 556         return status
 557
 558     def start_nodes (self, options):
 559         utils.header("Starting  nodes")
 560         for site_spec in self.plc_spec['sites']:
 561             TestSite(self,site_spec).start_nodes (options)
 562         return True
 563
 564     def stop_nodes (self, options):
 565         self.kill_all_qemus(options)
 566         return True
 567
 568     def check_tcp (self, options):
 569             #we just need to create a sliver object nothing else
 570             test_sliver=TestSliver(self,
 571                                    TestNode(self, TestSite(self,self.plc_spec['sites'][0]),
 572                                             self.plc_spec['sites'][0]['nodes'][0]),
 573                                    TestSlice(self,TestSite(self,self.plc_spec['sites'][0]),
 574                                              self.plc_spec['slices']))
 575             return test_sliver.do_check_tcp(self.plc_spec['tcp_param'],options)
 576
 577     # returns the filename to use for sql dump/restore, using options.dbname if set
 578     def dbfile (self, database, options):
 579         # uses options.dbname if it is found
 580         try:
 581             name=options.dbname
 582             if not isinstance(name,StringTypes):
 583                 raise Exception
 584         except:
 585             t=datetime.datetime.now()
 586             d=t.date()
 587             name=str(d)
 588         return "/root/%s-%s.sql"%(database,name)
 589
 590     def db_dump(self, options):
 591
 592         dump=self.dbfile("planetab4",options)
 593         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 594         utils.header('Dumped planetlab4 database in %s'%dump)
 595         return True
 596
 597     def db_restore(self, options):
 598         dump=self.dbfile("planetab4",options)
 599         ##stop httpd service
 600         self.run_in_guest('service httpd stop')
 601         # xxx - need another wrapper
 602         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 603         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 604         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 605         ##starting httpd service
 606         self.run_in_guest('service httpd start')
 607
 608         utils.header('Database restored from ' + dump)
 609
 610     @standby_generic
 611     def standby_1(): pass
 612     @standby_generic
 613     def standby_2(): pass
 614     @standby_generic
 615     def standby_3(): pass
 616     @standby_generic
 617     def standby_4(): pass
 618     @standby_generic
 619     def standby_5(): pass
 620     @standby_generic
 621     def standby_6(): pass
 622     @standby_generic
 623     def standby_7(): pass
 624     @standby_generic
 625     def standby_8(): pass
 626     @standby_generic
 627     def standby_9(): pass
 628     @standby_generic
 629     def standby_10(): pass
 630     @standby_generic
 631     def standby_11(): pass
 632     @standby_generic
 633     def standby_12(): pass
 634     @standby_generic
 635     def standby_13(): pass
 636     @standby_generic
 637     def standby_14(): pass
 638     @standby_generic
 639     def standby_15(): pass
 640     @standby_generic
 641     def standby_16(): pass
 642     @standby_generic
 643     def standby_17(): pass
 644     @standby_generic
 645     def standby_18(): pass
 646     @standby_generic
 647     def standby_19(): pass
 648     @standby_generic
 649     def standby_20(): pass
 650