system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16 from TestSlice import TestSlice
  17
  18 # inserts a backslash before each occurence of the following chars
  19 # \ " ' < > & | ; ( ) $ * ~
  20 def backslash_shell_specials (command):
  21     result=''
  22     for char in command:
  23         if char in "\\\"'<>&|;()$*~":
  24             result +='\\'+char
  25         else:
  26             result +=char
  27     return result
  28
  29 # step methods must take (self, options) and return a boolean
  30
  31 def standby(minutes):
  32         utils.header('Entering StandBy for %d mn'%minutes)
  33         time.sleep(60*minutes)
  34         return True
  35
  36 def standby_generic (func):
  37     def actual(self,options):
  38         minutes=int(func.__name__.split("_")[1])
  39         return standby(minutes)
  40     return actual
  41
  42 class TestPlc:
  43
  44     def __init__ (self,plc_spec):
  45         self.plc_spec=plc_spec
  46         self.path=os.path.dirname(sys.argv[0])
  47         try:
  48             self.vserverip=plc_spec['vserverip']
  49             self.vservername=plc_spec['vservername']
  50             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  51             self.vserver=True
  52         except:
  53             self.vserver=False
  54             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  55         utils.header('Using API url %s'%self.url)
  56         self.server=xmlrpclib.Server(self.url,allow_none=True)
  57
  58     def name(self):
  59         name=self.plc_spec['name']
  60         if self.vserver:
  61             return name+"[%s]"%self.vservername
  62         else:
  63             return name+"[chroot]"
  64
  65     def is_local (self):
  66         return self.plc_spec['hostname'] == 'localhost'
  67
  68     # define the API methods on this object through xmlrpc
  69     # would help, but not strictly necessary
  70     def connect (self):
  71         pass
  72
  73     # command gets run in the chroot/vserver
  74     def host_to_guest(self,command):
  75         if self.vserver:
  76             return "vserver %s exec %s"%(self.vservername,command)
  77         else:
  78             return "chroot /plc/root %s"%backslash_shell_specials(command)
  79
  80     # command gets run on the right box
  81     def to_host(self,command):
  82         if self.is_local():
  83             return command
  84         else:
  85             return "ssh %s %s"%(self.plc_spec['hostname'],backslash_shell_specials(command))
  86
  87     def full_command(self,command):
  88         return self.to_host(self.host_to_guest(command))
  89
  90     def run_in_guest (self,command):
  91         return utils.system(self.full_command(command))
  92     def run_in_host (self,command):
  93         return utils.system(self.to_host(command))
  94
  95     # xxx quick n dirty
  96     def run_in_guest_piped (self,local,remote):
  97         return utils.system(local+" | "+self.full_command(remote))
  98
  99     # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
 100     def copy_in_guest (self, localfile, remotefile, in_data=False):
 101         if in_data:
 102             chroot_dest="/plc/data"
 103         else:
 104             chroot_dest="/plc/root"
 105         if self.is_local():
 106             if not self.vserver:
 107                 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
 108             else:
 109                 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
 110         else:
 111             if not self.vserver:
 112                 utils.system("scp %s %s:%s/%s"%(localfile,self.plc_spec['hostname'],chroot_dest,remotefile))
 113             else:
 114                 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.plc_spec['hostname'],self.vservername,remotefile))
 115
 116     def auth_root (self):
 117         return {'Username':self.plc_spec['PLC_ROOT_USER'],
 118                 'AuthMethod':'password',
 119                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
 120                 'Role' : self.plc_spec['role']
 121                 }
 122     def locate_site (self,sitename):
 123         for site in self.plc_spec['sites']:
 124             if site['site_fields']['name'] == sitename:
 125                 return site
 126             if site['site_fields']['login_base'] == sitename:
 127                 return site
 128         raise Exception,"Cannot locate site %s"%sitename
 129
 130     def locate_node (self,nodename):
 131         for site in self.plc_spec['sites']:
 132             for node in site['nodes']:
 133                 if node['node_fields']['hostname'] == nodename:
 134                     return (site,node)
 135         raise Exception,"Cannot locate node %s"%nodename
 136
 137     def locate_key (self,keyname):
 138         for key in self.plc_spec['keys']:
 139             if key['name'] == keyname:
 140                 return key
 141         raise Exception,"Cannot locate key %s"%keyname
 142
 143     #this to catch up all different hostboxes used in this plc
 144     def locate_hostBoxes(self,site_spec):
 145         #Get The first host box to avoid returning a long list with the same host box
 146         #in case  only one is used for all the nodes
 147         HostBoxes=[site_spec['nodes'][0]['host_box']]
 148         for node_spec in site_spec['nodes']:
 149             if node_spec['host_box']!= HostBoxes[0]:
 150                 HostBoxes.append( node_spec['host_box'])
 151
 152         return HostBoxes
 153
 154     def kill_all_qemus(self):
 155         for site_spec in self.plc_spec['sites']:
 156             test_site = TestSite (self,site_spec)
 157             hostboxes_list=self.locate_hostBoxes(site_spec)
 158             if (hostboxes_list):
 159                 for node_spec in site_spec['nodes']:
 160                     TestNode(self,test_site,node_spec).stop_qemu(node_spec)
 161             else:
 162                 utils.header("No emulated node running on this PLC config ignore the kill() step")
 163
 164     def clear_ssh_config (self,options):
 165         # install local ssh_config file as root's .ssh/config - ssh should be quiet
 166         # dir might need creation first
 167         self.run_in_guest("mkdir /root/.ssh")
 168         self.run_in_guest("chmod 700 /root/.ssh")
 169         # this does not work - > redirection somehow makes it until an argument to cat
 170         #self.run_in_guest_piped("cat ssh_config","cat > /root/.ssh/config")
 171         self.copy_in_guest("ssh_config","/root/.ssh/config",True)
 172         return True
 173
 174     #################### step methods
 175
 176     ### uninstall
 177     def uninstall_chroot(self,options):
 178         self.run_in_host('service plc safestop')
 179         #####detecting the last myplc version installed and remove it
 180         self.run_in_host('rpm -e myplc')
 181         ##### Clean up the /plc directory
 182         self.run_in_host('rm -rf  /plc/data')
 183         ##### stop any running vservers
 184         self.run_in_host('for vserver in $(ls /vservers/* | sed -e s,/vservers/,,) ; do vserver $vserver stop ; done')
 185         return True
 186
 187     def uninstall_vserver(self,options):
 188         self.run_in_host("vserver --silent %s delete"%self.vservername)
 189         return True
 190
 191     def uninstall(self,options):
 192         # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
 193         # it sounds safer to have the former uninstalled too
 194         # now the vserver method cannot be invoked for chroot instances as vservername is required
 195         if self.vserver:
 196             self.uninstall_vserver(options)
 197             self.uninstall_chroot(options)
 198         else:
 199             self.uninstall_chroot(options)
 200         return True
 201
 202     ### install
 203     def install_chroot(self,options):
 204         # nothing to do
 205         return True
 206
 207     # xxx this would not work with hostname != localhost as mylc-init-vserver was extracted locally
 208     def install_vserver(self,options):
 209         # we need build dir for vtest-init-vserver
 210         if self.is_local():
 211             # a full path for the local calls
 212             build_dir=self.path+"/build"
 213         else:
 214             # use a standard name - will be relative to HOME
 215             build_dir="tests-system-build"
 216         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 217         if self.run_in_host(build_checkout) != 0:
 218             raise Exception,"Cannot checkout build dir"
 219         # the repo url is taken from myplc-url
 220         # with the last two steps (i386/myplc...) removed
 221         repo_url = options.myplc_url
 222         repo_url = os.path.dirname(repo_url)
 223         repo_url = os.path.dirname(repo_url)
 224         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 225             (build_dir,self.vservername,repo_url,self.vserverip)
 226         if self.run_in_host(create_vserver) != 0:
 227             raise Exception,"Could not create vserver for %s"%self.vservername
 228         return True
 229
 230     def install(self,options):
 231         if self.vserver:
 232             return self.install_vserver(options)
 233         else:
 234             return self.install_chroot(options)
 235
 236     ### install_rpm
 237     def install_rpm_chroot(self,options):
 238         utils.header('Installing from %s'%options.myplc_url)
 239         url=options.myplc_url
 240         self.run_in_host('rpm -Uvh '+url)
 241         self.run_in_host('service plc mount')
 242         return True
 243
 244     def install_rpm_vserver(self,options):
 245         self.run_in_guest("yum -y install myplc-native")
 246         return True
 247
 248     def install_rpm(self,options):
 249         if self.vserver:
 250             return self.install_rpm_vserver(options)
 251         else:
 252             return self.install_rpm_chroot(options)
 253
 254     ###
 255     def configure(self,options):
 256         tmpname='%s/%s.plc-config-tty'%(options.path,self.name())
 257         fileconf=open(tmpname,'w')
 258         for var in [ 'PLC_NAME',
 259                      'PLC_ROOT_PASSWORD',
 260                      'PLC_ROOT_USER',
 261                      'PLC_MAIL_ENABLED',
 262                      'PLC_MAIL_SUPPORT_ADDRESS',
 263                      'PLC_DB_HOST',
 264                      'PLC_API_HOST',
 265                      'PLC_WWW_HOST',
 266                      'PLC_BOOT_HOST',
 267                      'PLC_NET_DNS1',
 268                      'PLC_NET_DNS2']:
 269             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 270         fileconf.write('w\n')
 271         fileconf.write('q\n')
 272         fileconf.close()
 273         utils.system('cat %s'%tmpname)
 274         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 275         utils.system('rm %s'%tmpname)
 276         return True
 277
 278     # the chroot install is slightly different to this respect
 279     def start(self, options):
 280         if self.vserver:
 281             self.run_in_guest('service plc start')
 282         else:
 283             self.run_in_host('service plc start')
 284         return True
 285
 286     def stop(self, options):
 287         if self.vserver:
 288             self.run_in_guest('service plc stop')
 289         else:
 290             self.run_in_host('service plc stop')
 291         return True
 292
 293     # could use a TestKey class
 294     def store_keys(self, options):
 295         for key_spec in self.plc_spec['keys']:
 296             TestKey(self,key_spec).store_key()
 297         return True
 298
 299     def clean_keys(self, options):
 300         utils.system("rm -rf %s/keys/"%self.path)
 301
 302     def sites (self,options):
 303         return self.do_sites(options)
 304
 305     def clean_sites (self,options):
 306         return self.do_sites(options,action="delete")
 307
 308     def do_sites (self,options,action="add"):
 309         for site_spec in self.plc_spec['sites']:
 310             test_site = TestSite (self,site_spec)
 311             if (action != "add"):
 312                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 313                 test_site.delete_site()
 314                 # deleted with the site
 315                 #test_site.delete_users()
 316                 continue
 317             else:
 318                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 319                 test_site.create_site()
 320                 test_site.create_users()
 321         return True
 322
 323     def nodes (self, options):
 324         return self.do_nodes(options)
 325     def clean_nodes (self, options):
 326         return self.do_nodes(options,action="delete")
 327
 328     def do_nodes (self, options,action="add"):
 329         for site_spec in self.plc_spec['sites']:
 330             test_site = TestSite (self,site_spec)
 331             if action != "add":
 332                 utils.header("Deleting nodes in site %s"%test_site.name())
 333                 for node_spec in site_spec['nodes']:
 334                     test_node=TestNode(self,test_site,node_spec)
 335                     utils.header("Deleting %s"%test_node.name())
 336                     test_node.delete_node()
 337             else:
 338                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 339                 for node_spec in site_spec['nodes']:
 340                     utils.show_spec('Creating node %s'%node_spec,node_spec)
 341                     test_node = TestNode (self,test_site,node_spec)
 342                     test_node.create_node ()
 343         return True
 344
 345     # create nodegroups if needed, and populate
 346     # no need for a clean_nodegroups if we are careful enough
 347     def nodegroups (self, options):
 348         # 1st pass to scan contents
 349         groups_dict = {}
 350         for site_spec in self.plc_spec['sites']:
 351             test_site = TestSite (self,site_spec)
 352             for node_spec in site_spec['nodes']:
 353                 test_node=TestNode (self,test_site,node_spec)
 354                 if node_spec.has_key('nodegroups'):
 355                     nodegroupnames=node_spec['nodegroups']
 356                     if isinstance(nodegroupnames,StringTypes):
 357                         nodegroupnames = [ nodegroupnames ]
 358                     for nodegroupname in nodegroupnames:
 359                         if not groups_dict.has_key(nodegroupname):
 360                             groups_dict[nodegroupname]=[]
 361                         groups_dict[nodegroupname].append(test_node.name())
 362         auth=self.auth_root()
 363         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 364             try:
 365                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 366             except:
 367                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 368             for node in group_nodes:
 369                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 370         return True
 371
 372     def all_hostnames (self) :
 373         hostnames = []
 374         for site_spec in self.plc_spec['sites']:
 375             hostnames += [ node_spec['node_fields']['hostname'] \
 376                            for node_spec in site_spec['nodes'] ]
 377         return hostnames
 378
 379     # gracetime : during the first <gracetime> minutes nothing gets printed
 380     def do_nodes_booted (self, minutes, gracetime=2):
 381         # compute timeout
 382         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 383         graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
 384         # the nodes that haven't checked yet - start with a full list and shrink over time
 385         tocheck = self.all_hostnames()
 386         utils.header("checking nodes %r"%tocheck)
 387         # create a dict hostname -> status
 388         status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
 389         while tocheck:
 390             # get their status
 391             tocheck_status=self.server.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
 392             # update status
 393             for array in tocheck_status:
 394                 hostname=array['hostname']
 395                 boot_state=array['boot_state']
 396                 if boot_state == 'boot':
 397                     utils.header ("%s has reached the 'boot' state"%hostname)
 398                 else:
 399                     # if it's a real node, never mind
 400                     (site_spec,node_spec)=self.locate_node(hostname)
 401                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 402                         utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
 403                         # let's cheat
 404                         boot_state = 'boot'
 405                     if datetime.datetime.now() > graceout:
 406                         utils.header ("%s still in '%s' state"%(hostname,boot_state))
 407                         graceout=datetime.datetime.now()+datetime.timedelta(1)
 408                 status[hostname] = boot_state
 409             # refresh tocheck
 410             tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
 411             if not tocheck:
 412                 return True
 413             if datetime.datetime.now() > timeout:
 414                 for hostname in tocheck:
 415                     utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
 416                 return False
 417             # otherwise, sleep for a while
 418             time.sleep(15)
 419         # only useful in empty plcs
 420         return True
 421
 422     def nodes_booted(self,options):
 423         return self.do_nodes_booted(minutes=5)
 424
 425     #to scan and store the nodes's public keys and avoid to ask for confirmation when  ssh
 426     def scan_publicKeys(self,hostnames):
 427         try:
 428             temp_knownhosts="/root/known_hosts"
 429             remote_knownhosts="/root/.ssh/known_hosts"
 430             self.run_in_host("touch %s"%temp_knownhosts )
 431             for hostname in hostnames:
 432                 utils.header("Scan Public %s key and store it in the known_host file(under the root image) "%hostname)
 433                 scan=self.run_in_host('ssh-keyscan -t rsa %s >> %s '%(hostname,temp_knownhosts))
 434             #Store the public keys in the right root image
 435             self.copy_in_guest(temp_knownhosts,remote_knownhosts,True)
 436             #clean the temp keys file used
 437             self.run_in_host('rm -f  %s '%temp_knownhosts )
 438         except Exception, err:
 439             print err
 440
 441     def do_check_nodesSsh(self,minutes):
 442         # compute timeout
 443         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 444         #graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
 445         tocheck = self.all_hostnames()
 446         self.scan_publicKeys(tocheck)
 447         utils.header("checking Connectivity on nodes %r"%tocheck)
 448         while tocheck:
 449             for hostname in tocheck:
 450                 # try to ssh in nodes
 451                 access=self.run_in_guest('ssh -i /etc/planetlab/root_ssh_key.rsa root@%s date'%hostname )
 452                 if (not access):
 453                     utils.header('The node %s is sshable -->'%hostname)
 454                     # refresh tocheck
 455                     tocheck.remove(hostname)
 456                 else:
 457                     (site_spec,node_spec)=self.locate_node(hostname)
 458                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 459                         utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
 460                     tocheck.remove(hostname)
 461             if not tocheck:
 462                 return True
 463             if datetime.datetime.now() > timeout:
 464                 for hostname in tocheck:
 465                     utils.header("FAILURE to ssh into %s"%hostname)
 466                 return False
 467             # otherwise, sleep for a while
 468             time.sleep(15)
 469         # only useful in empty plcs
 470         return True
 471
 472     def nodes_ssh(self, options):
 473         return  self.do_check_nodesSsh(minutes=2)
 474
 475     def bootcd (self, options):
 476         for site_spec in self.plc_spec['sites']:
 477             test_site = TestSite (self,site_spec)
 478             for node_spec in site_spec['nodes']:
 479                 test_node=TestNode (self,test_site,node_spec)
 480                 test_node.create_boot_cd(options.path)
 481         return True
 482
 483     def initscripts (self, options):
 484         for initscript in self.plc_spec['initscripts']:
 485             utils.show_spec('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 486             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 487         return True
 488
 489     def slices (self, options):
 490         return self.do_slices()
 491
 492     def clean_slices (self, options):
 493         return self.do_slices("delete")
 494
 495     def do_slices (self,  action="add"):
 496         for slice in self.plc_spec['slices']:
 497             site_spec = self.locate_site (slice['sitename'])
 498             test_site = TestSite(self,site_spec)
 499             test_slice=TestSlice(self,test_site,slice)
 500             if action != "add":
 501                 utils.header("Deleting slices in site %s"%test_site.name())
 502                 test_slice.delete_slice()
 503             else:
 504                 utils.show_spec("Creating slice",slice)
 505                 test_slice.create_slice()
 506                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
 507         return True
 508
 509     def check_slices(self, options):
 510         for slice_spec in self.plc_spec['slices']:
 511             site_spec = self.locate_site (slice_spec['sitename'])
 512             test_site = TestSite(self,site_spec)
 513             test_slice=TestSlice(self,test_site,slice_spec)
 514             status=test_slice.do_check_slice(options)
 515             if (not status):
 516                 return False
 517         return status
 518
 519     def start_nodes (self, options):
 520         self.kill_all_qemus()
 521         utils.header("Starting  nodes")
 522         for site_spec in self.plc_spec['sites']:
 523             TestSite(self,site_spec).start_nodes (options)
 524         return True
 525
 526     def stop_nodes (self, options):
 527         self.kill_all_qemus()
 528         return True
 529
 530     # returns the filename to use for sql dump/restore, using options.dbname if set
 531     def dbfile (self, database, options):
 532         # uses options.dbname if it is found
 533         try:
 534             name=options.dbname
 535             if not isinstance(name,StringTypes):
 536                 raise Exception
 537         except:
 538             t=datetime.datetime.now()
 539             d=t.date()
 540             name=str(d)
 541         return "/root/%s-%s.sql"%(database,name)
 542
 543     def db_dump(self, options):
 544
 545         dump=self.dbfile("planetab4",options)
 546         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 547         utils.header('Dumped planetlab4 database in %s'%dump)
 548         return True
 549
 550     def db_restore(self, options):
 551         dump=self.dbfile("planetab4",options)
 552         ##stop httpd service
 553         self.run_in_guest('service httpd stop')
 554         # xxx - need another wrapper
 555         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 556         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 557         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 558         ##starting httpd service
 559         self.run_in_guest('service httpd start')
 560
 561         utils.header('Database restored from ' + dump)
 562
 563     @standby_generic
 564     def standby_1(): pass
 565     @standby_generic
 566     def standby_2(): pass
 567     @standby_generic
 568     def standby_3(): pass
 569     @standby_generic
 570     def standby_4(): pass
 571     @standby_generic
 572     def standby_5(): pass
 573     @standby_generic
 574     def standby_6(): pass
 575     @standby_generic
 576     def standby_7(): pass
 577     @standby_generic
 578     def standby_8(): pass
 579     @standby_generic
 580     def standby_9(): pass
 581     @standby_generic
 582     def standby_10(): pass
 583     @standby_generic
 584     def standby_11(): pass
 585     @standby_generic
 586     def standby_12(): pass
 587     @standby_generic
 588     def standby_13(): pass
 589     @standby_generic
 590     def standby_14(): pass
 591     @standby_generic
 592     def standby_15(): pass
 593     @standby_generic
 594     def standby_16(): pass
 595     @standby_generic
 596     def standby_17(): pass
 597     @standby_generic
 598     def standby_18(): pass
 599     @standby_generic
 600     def standby_19(): pass
 601     @standby_generic
 602     def standby_20(): pass
 603