6c2195bb883841ff7489f5f977c6b39eb26bb886
[tests.git] / system / TestPlc.py
1 # $Id$
2 import os, os.path
3 import datetime
4 import time
5 import sys
6 import xmlrpclib
7 import datetime
8 import traceback
9 from types import StringTypes
10
11 import utils
12 from TestSite import TestSite
13 from TestNode import TestNode
14 from TestUser import TestUser
15 from TestKey import TestKey
16 from TestSlice import TestSlice
17 from TestSliver import TestSliver
18 from TestBox import TestBox
19
20 # step methods must take (self, options) and return a boolean
21
22 def standby(minutes):
23         utils.header('Entering StandBy for %d mn'%minutes)
24         time.sleep(60*minutes)
25         return True
26
27 def standby_generic (func):
28     def actual(self,options):
29         minutes=int(func.__name__.split("_")[1])
30         return standby(minutes)
31     return actual
32
33 class TestPlc:
34
35     def __init__ (self,plc_spec):
36         self.plc_spec=plc_spec
37         self.path=os.path.dirname(sys.argv[0])
38         try:
39             self.vserverip=plc_spec['vserverip']
40             self.vservername=plc_spec['vservername']
41             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
42             self.vserver=True
43         except:
44             self.vserver=False
45             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
46         utils.header('Using API url %s'%self.url)
47         self.server=xmlrpclib.Server(self.url,allow_none=True)
48         
49     def name(self):
50         name=self.plc_spec['name']
51         if self.vserver:
52             return name+"[%s]"%self.vservername
53         else:
54             return name+"[chroot]"
55
56     def hostname(self):
57         return self.plc_spec['hostname']
58
59     def is_local (self):
60         return utils.is_local(self.hostname())
61
62     # define the API methods on this object through xmlrpc
63     # would help, but not strictly necessary
64     def connect (self):
65         pass
66     
67     # command gets run in the chroot/vserver
68     def host_to_guest(self,command):
69         if self.vserver:
70             return "vserver %s exec %s"%(self.vservername,command)
71         else:
72             return "chroot /plc/root %s"%utils.backslash_shell_specials(command)
73
74     # command gets run on the right box
75     def to_host(self,command):
76         if self.is_local():
77             return command
78         else:
79             return "ssh %s %s"%(self.hostname(),utils.backslash_shell_specials(command))
80
81     def full_command(self,command):
82         return self.to_host(self.host_to_guest(command))
83
84     def run_in_guest (self,command):
85         return utils.system(self.full_command(command))
86     def run_in_host (self,command):
87         return utils.system(self.to_host(command))
88
89     # xxx quick n dirty
90     def run_in_guest_piped (self,local,remote):
91         return utils.system(local+" | "+self.full_command(remote))
92
93     # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
94     def copy_in_guest (self, localfile, remotefile, in_data=False):
95         if in_data:
96             chroot_dest="/plc/data"
97         else:
98             chroot_dest="/plc/root"
99         if self.is_local():
100             if not self.vserver:
101                 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
102             else:
103                 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
104         else:
105             if not self.vserver:
106                 utils.system("scp %s %s:%s/%s"%(localfile,self.hostname(),chroot_dest,remotefile))
107             else:
108                 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.hostname(),self.vservername,remotefile))
109
110     def auth_root (self):
111         return {'Username':self.plc_spec['PLC_ROOT_USER'],
112                 'AuthMethod':'password',
113                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
114                 'Role' : self.plc_spec['role']
115                 }
116     def locate_site (self,sitename):
117         for site in self.plc_spec['sites']:
118             if site['site_fields']['name'] == sitename:
119                 return site
120             if site['site_fields']['login_base'] == sitename:
121                 return site
122         raise Exception,"Cannot locate site %s"%sitename
123         
124     def locate_node (self,nodename):
125         for site in self.plc_spec['sites']:
126             for node in site['nodes']:
127                 if node['node_fields']['hostname'] == nodename:
128                     return (site,node)
129         raise Exception,"Cannot locate node %s"%nodename
130         
131     def locate_key (self,keyname):
132         for key in self.plc_spec['keys']:
133             if key['name'] == keyname:
134                 return key
135         raise Exception,"Cannot locate key %s"%keyname
136
137     # all different hostboxes used in this plc
138     def gather_hostBoxes(self):
139         # maps on sites and nodes, return [ (host_box,test_node) ]
140         tuples=[]
141         for site_spec in self.plc_spec['sites']:
142             test_site = TestSite (self,site_spec)
143             for node_spec in site_spec['nodes']:
144                 test_node = TestNode (self, test_site, node_spec)
145                 if not test_node.is_real():
146                     tuples.append( (test_node.host_box(),test_node) )
147         # transform into a dict { 'host_box' -> [ hostnames .. ] }
148         result = {}
149         for (box,node) in tuples:
150             if not result.has_key(box):
151                 result[box]=[node]
152             else:
153                 result[box].append(node)
154         return result
155                     
156     # a step for checking this stuff
157     def showboxes (self,options):
158         print 'showboxes'
159         for (box,nodes) in self.gather_hostBoxes().iteritems():
160             print box,":"," + ".join( [ node.name() for node in nodes ] )
161         return True
162
163     # make this a valid step
164     def kill_all_qemus(self,options):
165         for (box,nodes) in self.gather_hostBoxes().iteritems():
166             # this is the brute force version, kill all qemus on that host box
167             TestBox(box,options.buildname).kill_all_qemus()
168         return True
169
170     # make this a valid step
171     def list_all_qemus(self,options):
172         for (box,nodes) in self.gather_hostBoxes().iteritems():
173             # push the script
174             TestBox(box,options.buildname).copy("qemu_kill.sh") 
175             # this is the brute force version, kill all qemus on that host box
176             TestBox(box,options.buildname).run_in_buildname("qemu_kill.sh -l")
177         return True
178
179     # kill only the right qemus
180     def kill_qemus(self,options):
181         for (box,nodes) in self.gather_hostBoxes().iteritems():
182             # push the script
183             TestBox(box,options.buildname).copy("qemu_kill.sh") 
184             # the fine-grain version
185             for node in nodes:
186                 node.kill_qemu()
187         return True
188
189     def clear_ssh_config (self,options):
190         # install local ssh_config file as root's .ssh/config - ssh should be quiet
191         # dir might need creation first
192         self.run_in_guest("mkdir /root/.ssh")
193         self.run_in_guest("chmod 700 /root/.ssh")
194         # this does not work - > redirection somehow makes it until an argument to cat
195         #self.run_in_guest_piped("cat ssh_config","cat > /root/.ssh/config")
196         self.copy_in_guest("ssh_config","/root/.ssh/config",True)
197         return True
198             
199     #################### step methods
200
201     ### uninstall
202     def uninstall_chroot(self,options):
203         self.run_in_host('service plc safestop')
204         #####detecting the last myplc version installed and remove it
205         self.run_in_host('rpm -e myplc')
206         ##### Clean up the /plc directory
207         self.run_in_host('rm -rf  /plc/data')
208         return True
209
210     def uninstall_vserver(self,options):
211         self.run_in_host("vserver --silent %s delete"%self.vservername)
212         return True
213
214     def stop_all_vservers (self,options):
215         ##### stop any running vservers
216         self.run_in_host('for vserver in $(ls -d /vservers/* | sed -e s,/vservers/,,) ; do echo Stopping $vserver; vserver $vserver stop ; done')
217         return True
218
219     def uninstall(self,options):
220         # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
221         # it sounds safer to have the former uninstalled too
222         # now the vserver method cannot be invoked for chroot instances as vservername is required
223         if self.vserver:
224             self.uninstall_vserver(options)
225             self.uninstall_chroot(options)
226         else:
227             self.uninstall_chroot(options)
228             self.stop_all_vservers(options)
229         return True
230
231     ### install
232     def install_chroot(self,options):
233         # nothing to do
234         return True
235
236     # xxx this would not work with hostname != localhost as mylc-init-vserver was extracted locally
237     def install_vserver(self,options):
238         # we need build dir for vtest-init-vserver
239         if self.is_local():
240             # a full path for the local calls
241             build_dir=self.path+"/build"
242         else:
243             # use a standard name - will be relative to HOME 
244             build_dir="tests-system-build"
245         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
246         if self.run_in_host(build_checkout) != 0:
247             raise Exception,"Cannot checkout build dir"
248         # the repo url is taken from myplc-url 
249         # with the last two steps (i386/myplc...) removed
250         repo_url = options.myplc_url
251         repo_url = os.path.dirname(repo_url)
252         repo_url = os.path.dirname(repo_url)
253         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
254             (build_dir,self.vservername,repo_url,self.vserverip)
255         if self.run_in_host(create_vserver) != 0:
256             raise Exception,"Could not create vserver for %s"%self.vservername
257         return True
258
259     def install(self,options):
260         if self.vserver:
261             return self.install_vserver(options)
262         else:
263             return self.install_chroot(options)
264     
265     ### install_rpm
266     def cache_rpm(self,url):
267         self.run_in_host('rm -rf *.rpm')
268         utils.header('Curling rpm from %s'%url)
269         id= self.run_in_host('curl -O '+url)
270         if (id != 0):
271                 raise Exception,"Could not get rpm from  %s"%url
272                 return False
273         return True
274
275     def install_rpm_chroot(self,options):
276         rpm = os.path.basename(options.myplc_url)
277         if (not os.path.isfile(rpm)):
278                 self.cache_rpm(options.myplc_url)
279         utils.header('Installing the :  %s'%rpm)
280         self.run_in_host('rpm -Uvh '+rpm)
281         self.run_in_host('service plc mount')
282         return True
283
284     def install_rpm_vserver(self,options):
285         self.run_in_guest("yum -y install myplc-native")
286         return True
287
288     def install_rpm(self,options):
289         if self.vserver:
290             return self.install_rpm_vserver(options)
291         else:
292             return self.install_rpm_chroot(options)
293
294     ### 
295     def configure(self,options):
296         tmpname='%s.plc-config-tty'%(self.name())
297         fileconf=open(tmpname,'w')
298         for var in [ 'PLC_NAME',
299                      'PLC_ROOT_PASSWORD',
300                      'PLC_ROOT_USER',
301                      'PLC_MAIL_ENABLED',
302                      'PLC_MAIL_SUPPORT_ADDRESS',
303                      'PLC_DB_HOST',
304                      'PLC_API_HOST',
305                      'PLC_WWW_HOST',
306                      'PLC_BOOT_HOST',
307                      'PLC_NET_DNS1',
308                      'PLC_NET_DNS2']:
309             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
310         fileconf.write('w\n')
311         fileconf.write('q\n')
312         fileconf.close()
313         utils.system('cat %s'%tmpname)
314         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
315         utils.system('rm %s'%tmpname)
316         return True
317
318     # the chroot install is slightly different to this respect
319     def start(self, options):
320         if self.vserver:
321             self.run_in_guest('service plc start')
322         else:
323             self.run_in_host('service plc start')
324         return True
325         
326     def stop(self, options):
327         if self.vserver:
328             self.run_in_guest('service plc stop')
329         else:
330             self.run_in_host('service plc stop')
331         return True
332         
333     # could use a TestKey class
334     def store_keys(self, options):
335         for key_spec in self.plc_spec['keys']:
336             TestKey(self,key_spec).store_key()
337         return True
338
339     def clean_keys(self, options):
340         utils.system("rm -rf %s/keys/"%self.path)
341
342     def sites (self,options):
343         return self.do_sites(options)
344     
345     def clean_sites (self,options):
346         return self.do_sites(options,action="delete")
347     
348     def do_sites (self,options,action="add"):
349         for site_spec in self.plc_spec['sites']:
350             test_site = TestSite (self,site_spec)
351             if (action != "add"):
352                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
353                 test_site.delete_site()
354                 # deleted with the site
355                 #test_site.delete_users()
356                 continue
357             else:
358                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
359                 test_site.create_site()
360                 test_site.create_users()
361         return True
362
363     def nodes (self, options):
364         return self.do_nodes(options)
365     def clean_nodes (self, options):
366         return self.do_nodes(options,action="delete")
367
368     def do_nodes (self, options,action="add"):
369         for site_spec in self.plc_spec['sites']:
370             test_site = TestSite (self,site_spec)
371             if action != "add":
372                 utils.header("Deleting nodes in site %s"%test_site.name())
373                 for node_spec in site_spec['nodes']:
374                     test_node=TestNode(self,test_site,node_spec)
375                     utils.header("Deleting %s"%test_node.name())
376                     test_node.delete_node()
377             else:
378                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
379                 for node_spec in site_spec['nodes']:
380                     utils.pprint('Creating node %s'%node_spec,node_spec)
381                     test_node = TestNode (self,test_site,node_spec)
382                     test_node.create_node ()
383         return True
384
385     # create nodegroups if needed, and populate
386     # no need for a clean_nodegroups if we are careful enough
387     def nodegroups (self, options):
388         # 1st pass to scan contents
389         groups_dict = {}
390         for site_spec in self.plc_spec['sites']:
391             test_site = TestSite (self,site_spec)
392             for node_spec in site_spec['nodes']:
393                 test_node=TestNode (self,test_site,node_spec)
394                 if node_spec.has_key('nodegroups'):
395                     nodegroupnames=node_spec['nodegroups']
396                     if isinstance(nodegroupnames,StringTypes):
397                         nodegroupnames = [ nodegroupnames ]
398                     for nodegroupname in nodegroupnames:
399                         if not groups_dict.has_key(nodegroupname):
400                             groups_dict[nodegroupname]=[]
401                         groups_dict[nodegroupname].append(test_node.name())
402         auth=self.auth_root()
403         for (nodegroupname,group_nodes) in groups_dict.iteritems():
404             try:
405                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
406             except:
407                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
408             for node in group_nodes:
409                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
410         return True
411
412     def all_hostnames (self) :
413         hostnames = []
414         for site_spec in self.plc_spec['sites']:
415             hostnames += [ node_spec['node_fields']['hostname'] \
416                            for node_spec in site_spec['nodes'] ]
417         return hostnames
418
419     # gracetime : during the first <gracetime> minutes nothing gets printed
420     def do_nodes_booted (self, minutes, gracetime=2):
421         # compute timeout
422         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
423         graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
424         # the nodes that haven't checked yet - start with a full list and shrink over time
425         tocheck = self.all_hostnames()
426         utils.header("checking nodes %r"%tocheck)
427         # create a dict hostname -> status
428         status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
429         while tocheck:
430             # get their status
431             tocheck_status=self.server.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
432             # update status
433             for array in tocheck_status:
434                 hostname=array['hostname']
435                 boot_state=array['boot_state']
436                 if boot_state == 'boot':
437                     utils.header ("%s has reached the 'boot' state"%hostname)
438                 else:
439                     # if it's a real node, never mind
440                     (site_spec,node_spec)=self.locate_node(hostname)
441                     if TestNode.is_real_model(node_spec['node_fields']['model']):
442                         utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
443                         # let's cheat
444                         boot_state = 'boot'
445                     if datetime.datetime.now() > graceout:
446                         utils.header ("%s still in '%s' state"%(hostname,boot_state))
447                         graceout=datetime.datetime.now()+datetime.timedelta(1)
448                 status[hostname] = boot_state
449             # refresh tocheck
450             tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
451             if not tocheck:
452                 return True
453             if datetime.datetime.now() > timeout:
454                 for hostname in tocheck:
455                     utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
456                 return False
457             # otherwise, sleep for a while
458             time.sleep(15)
459         # only useful in empty plcs
460         return True
461
462     def nodes_booted(self,options):
463         return self.do_nodes_booted(minutes=5)
464     
465     #to scan and store the nodes's public keys and avoid to ask for confirmation when  ssh 
466     def scan_publicKeys(self,hostnames):
467         try:
468             temp_knownhosts="/root/known_hosts"
469             remote_knownhosts="/root/.ssh/known_hosts"
470             self.run_in_host("touch %s"%temp_knownhosts )
471             for hostname in hostnames:
472                 utils.header("Scan Public %s key and store it in the known_host file(under the root image) "%hostname)
473                 scan=self.run_in_host('ssh-keyscan -t rsa %s >> %s '%(hostname,temp_knownhosts))
474             #Store the public keys in the right root image
475             self.copy_in_guest(temp_knownhosts,remote_knownhosts,True)
476             #clean the temp keys file used
477             self.run_in_host('rm -f  %s '%temp_knownhosts )
478         except Exception, err:
479             print err
480             
481     def do_check_nodesSsh(self,minutes):
482         # compute timeout
483         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
484         tocheck = self.all_hostnames()
485         self.scan_publicKeys(tocheck)
486         utils.header("checking Connectivity on nodes %r"%tocheck)
487         while tocheck:
488             for hostname in tocheck:
489                 # try to ssh in nodes
490                 access=self.run_in_guest('ssh -i /etc/planetlab/root_ssh_key.rsa root@%s date'%hostname )
491                 if (not access):
492                     utils.header('The node %s is sshable -->'%hostname)
493                     # refresh tocheck
494                     tocheck.remove(hostname)
495                 else:
496                     (site_spec,node_spec)=self.locate_node(hostname)
497                     if TestNode.is_real_model(node_spec['node_fields']['model']):
498                         utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
499                         tocheck.remove(hostname)
500             if  not tocheck:
501                 return True
502             if datetime.datetime.now() > timeout:
503                 for hostname in tocheck:
504                     utils.header("FAILURE to ssh into %s"%hostname)
505                 return False
506             # otherwise, sleep for a while
507             time.sleep(15)
508         # only useful in empty plcs
509         return True
510         
511     def nodes_ssh(self, options):
512         return  self.do_check_nodesSsh(minutes=2)
513     
514     def bootcd (self, options):
515         for site_spec in self.plc_spec['sites']:
516             test_site = TestSite (self,site_spec)
517             for node_spec in site_spec['nodes']:
518                 test_node=TestNode (self,test_site,node_spec)
519                 test_node.prepare_area()
520                 test_node.create_boot_cd()
521                 test_node.configure_qemu()
522         return True
523
524     def do_check_intiscripts(self):
525         for site_spec in self.plc_spec['sites']:
526                 test_site = TestSite (self,site_spec)
527                 test_node = TestNode (self,test_site,site_spec['nodes'])
528                 for slice_spec in self.plc_spec['slices']:
529                         test_slice=TestSlice (self,test_site,slice_spec)
530                         test_sliver=TestSliver(self,test_node,test_slice)
531                         init_status=test_sliver.get_initscript(slice_spec)
532                         if (not init_status):
533                                 return False
534                 return init_status
535             
536     def check_initscripts(self, options):
537             return self.do_check_intiscripts()
538                     
539     def initscripts (self, options):
540         for initscript in self.plc_spec['initscripts']:
541             utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
542             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
543         return True
544
545     def slices (self, options):
546         return self.do_slices()
547
548     def clean_slices (self, options):
549         return self.do_slices("delete")
550
551     def do_slices (self,  action="add"):
552         for slice in self.plc_spec['slices']:
553             site_spec = self.locate_site (slice['sitename'])
554             test_site = TestSite(self,site_spec)
555             test_slice=TestSlice(self,test_site,slice)
556             if action != "add":
557                 utils.header("Deleting slices in site %s"%test_site.name())
558                 test_slice.delete_slice()
559             else:    
560                 utils.pprint("Creating slice",slice)
561                 test_slice.create_slice()
562                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
563         return True
564         
565     def check_slices(self, options):
566         for slice_spec in self.plc_spec['slices']:
567             site_spec = self.locate_site (slice_spec['sitename'])
568             test_site = TestSite(self,site_spec)
569             test_slice=TestSlice(self,test_site,slice_spec)
570             status=test_slice.do_check_slice(options)
571             if (not status):
572                 return False
573         return status
574     
575     def start_nodes (self, options):
576         utils.header("Starting  nodes")
577         for site_spec in self.plc_spec['sites']:
578             TestSite(self,site_spec).start_nodes (options)
579         return True
580
581     def stop_nodes (self, options):
582         self.kill_all_qemus(options)
583         return True
584
585     def check_tcp (self, options):
586             #we just need to create a sliver object nothing else
587             test_sliver=TestSliver(self,
588                                    TestNode(self, TestSite(self,self.plc_spec['sites'][0]),
589                                             self.plc_spec['sites'][0]['nodes'][0]),
590                                    TestSlice(self,TestSite(self,self.plc_spec['sites'][0]),
591                                              self.plc_spec['slices']))
592             return test_sliver.do_check_tcp(self.plc_spec['tcp_param'],options)
593
594     # returns the filename to use for sql dump/restore, using options.dbname if set
595     def dbfile (self, database, options):
596         # uses options.dbname if it is found
597         try:
598             name=options.dbname
599             if not isinstance(name,StringTypes):
600                 raise Exception
601         except:
602             t=datetime.datetime.now()
603             d=t.date()
604             name=str(d)
605         return "/root/%s-%s.sql"%(database,name)
606
607     def db_dump(self, options):
608         
609         dump=self.dbfile("planetab4",options)
610         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
611         utils.header('Dumped planetlab4 database in %s'%dump)
612         return True
613
614     def db_restore(self, options):
615         dump=self.dbfile("planetab4",options)
616         ##stop httpd service
617         self.run_in_guest('service httpd stop')
618         # xxx - need another wrapper
619         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
620         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
621         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
622         ##starting httpd service
623         self.run_in_guest('service httpd start')
624
625         utils.header('Database restored from ' + dump)
626
627     @standby_generic 
628     def standby_1(): pass
629     @standby_generic 
630     def standby_2(): pass
631     @standby_generic 
632     def standby_3(): pass
633     @standby_generic 
634     def standby_4(): pass
635     @standby_generic 
636     def standby_5(): pass
637     @standby_generic 
638     def standby_6(): pass
639     @standby_generic 
640     def standby_7(): pass
641     @standby_generic 
642     def standby_8(): pass
643     @standby_generic 
644     def standby_9(): pass
645     @standby_generic 
646     def standby_10(): pass
647     @standby_generic 
648     def standby_11(): pass
649     @standby_generic 
650     def standby_12(): pass
651     @standby_generic 
652     def standby_13(): pass
653     @standby_generic 
654     def standby_14(): pass
655     @standby_generic 
656     def standby_15(): pass
657     @standby_generic 
658     def standby_16(): pass
659     @standby_generic 
660     def standby_17(): pass
661     @standby_generic 
662     def standby_18(): pass
663     @standby_generic 
664     def standby_19(): pass
665     @standby_generic 
666     def standby_20(): pass
667