system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16 from TestSlice import TestSlice
  17 from TestBox import TestBox
  18
  19 # inserts a backslash before each occurence of the following chars
  20 # \ " ' < > & | ; ( ) $ * ~
  21 def backslash_shell_specials (command):
  22     result=''
  23     for char in command:
  24         if char in "\\\"'<>&|;()$*~":
  25             result +='\\'+char
  26         else:
  27             result +=char
  28     return result
  29
  30 # step methods must take (self, options) and return a boolean
  31
  32 def standby(minutes):
  33         utils.header('Entering StandBy for %d mn'%minutes)
  34         time.sleep(60*minutes)
  35         return True
  36
  37 def standby_generic (func):
  38     def actual(self,options):
  39         minutes=int(func.__name__.split("_")[1])
  40         return standby(minutes)
  41     return actual
  42
  43 class TestPlc:
  44
  45     def __init__ (self,plc_spec):
  46         self.plc_spec=plc_spec
  47         self.path=os.path.dirname(sys.argv[0])
  48         try:
  49             self.vserverip=plc_spec['vserverip']
  50             self.vservername=plc_spec['vservername']
  51             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  52             self.vserver=True
  53         except:
  54             self.vserver=False
  55             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  56         utils.header('Using API url %s'%self.url)
  57         self.server=xmlrpclib.Server(self.url,allow_none=True)
  58
  59     def name(self):
  60         name=self.plc_spec['name']
  61         if self.vserver:
  62             return name+"[%s]"%self.vservername
  63         else:
  64             return name+"[chroot]"
  65
  66     def is_local (self):
  67         return self.plc_spec['hostname'] == 'localhost'
  68
  69     # define the API methods on this object through xmlrpc
  70     # would help, but not strictly necessary
  71     def connect (self):
  72         pass
  73
  74     # command gets run in the chroot/vserver
  75     def host_to_guest(self,command):
  76         if self.vserver:
  77             return "vserver %s exec %s"%(self.vservername,command)
  78         else:
  79             return "chroot /plc/root %s"%backslash_shell_specials(command)
  80
  81     # command gets run on the right box
  82     def to_host(self,command):
  83         if self.is_local():
  84             return command
  85         else:
  86             return "ssh %s %s"%(self.plc_spec['hostname'],backslash_shell_specials(command))
  87
  88     def full_command(self,command):
  89         return self.to_host(self.host_to_guest(command))
  90
  91     def run_in_guest (self,command):
  92         return utils.system(self.full_command(command))
  93     def run_in_host (self,command):
  94         return utils.system(self.to_host(command))
  95
  96     # xxx quick n dirty
  97     def run_in_guest_piped (self,local,remote):
  98         return utils.system(local+" | "+self.full_command(remote))
  99
 100     # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
 101     def copy_in_guest (self, localfile, remotefile, in_data=False):
 102         if in_data:
 103             chroot_dest="/plc/data"
 104         else:
 105             chroot_dest="/plc/root"
 106         if self.is_local():
 107             if not self.vserver:
 108                 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
 109             else:
 110                 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
 111         else:
 112             if not self.vserver:
 113                 utils.system("scp %s %s:%s/%s"%(localfile,self.plc_spec['hostname'],chroot_dest,remotefile))
 114             else:
 115                 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.plc_spec['hostname'],self.vservername,remotefile))
 116
 117     def auth_root (self):
 118         return {'Username':self.plc_spec['PLC_ROOT_USER'],
 119                 'AuthMethod':'password',
 120                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
 121                 'Role' : self.plc_spec['role']
 122                 }
 123     def locate_site (self,sitename):
 124         for site in self.plc_spec['sites']:
 125             if site['site_fields']['name'] == sitename:
 126                 return site
 127             if site['site_fields']['login_base'] == sitename:
 128                 return site
 129         raise Exception,"Cannot locate site %s"%sitename
 130
 131     def locate_node (self,nodename):
 132         for site in self.plc_spec['sites']:
 133             for node in site['nodes']:
 134                 if node['node_fields']['hostname'] == nodename:
 135                     return (site,node)
 136         raise Exception,"Cannot locate node %s"%nodename
 137
 138     def locate_key (self,keyname):
 139         for key in self.plc_spec['keys']:
 140             if key['name'] == keyname:
 141                 return key
 142         raise Exception,"Cannot locate key %s"%keyname
 143
 144     # all different hostboxes used in this plc
 145     def gather_hostBoxes(self):
 146         # maps on sites and nodes, return [ (host_box,hostname) ]
 147         tuples=[]
 148         for site_spec in self.plc_spec['sites']:
 149             test_site = TestSite (self,site_spec)
 150             for node_spec in site_spec['nodes']:
 151                 test_node = TestNode (self, test_site, node_spec)
 152                 if not test_node.is_real():
 153                     tuples.append( (test_node.host_box(),node_spec['node_fields']['hostname']) )
 154         # transform into a dict { 'host_box' -> [ hostnames .. ] }
 155         result = {}
 156         for (box,hostname) in tuples:
 157             if not result.has_key(box):
 158                 result[box]=[hostname]
 159             else:
 160                 result[box].append(hostname)
 161         return result
 162
 163     # a step for checking this stuff
 164     def showboxes (self,options):
 165         print 'showboxes'
 166         for (box,hosts) in self.gather_hostBoxes().iteritems():
 167             print box,":"," + ".join(hosts)
 168         return True
 169
 170     def kill_all_qemus(self):
 171         for (box,hosts) in self.gather_hostBoxes().iteritems():
 172             # this is the brute force version, kill all qemus on that host box
 173             TestBox(box).kill_all_qemus()
 174
 175     def clear_ssh_config (self,options):
 176         # install local ssh_config file as root's .ssh/config - ssh should be quiet
 177         # dir might need creation first
 178         self.run_in_guest("mkdir /root/.ssh")
 179         self.run_in_guest("chmod 700 /root/.ssh")
 180         # this does not work - > redirection somehow makes it until an argument to cat
 181         #self.run_in_guest_piped("cat ssh_config","cat > /root/.ssh/config")
 182         self.copy_in_guest("ssh_config","/root/.ssh/config",True)
 183         return True
 184
 185     #################### step methods
 186
 187     ### uninstall
 188     def uninstall_chroot(self,options):
 189         self.run_in_host('service plc safestop')
 190         #####detecting the last myplc version installed and remove it
 191         self.run_in_host('rpm -e myplc')
 192         ##### Clean up the /plc directory
 193         self.run_in_host('rm -rf  /plc/data')
 194         ##### stop any running vservers
 195         self.run_in_host('for vserver in $(ls /vservers/* | sed -e s,/vservers/,,) ; do vserver $vserver stop ; done')
 196         return True
 197
 198     def uninstall_vserver(self,options):
 199         self.run_in_host("vserver --silent %s delete"%self.vservername)
 200         return True
 201
 202     def uninstall(self,options):
 203         # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
 204         # it sounds safer to have the former uninstalled too
 205         # now the vserver method cannot be invoked for chroot instances as vservername is required
 206         if self.vserver:
 207             self.uninstall_vserver(options)
 208             self.uninstall_chroot(options)
 209         else:
 210             self.uninstall_chroot(options)
 211         return True
 212
 213     ### install
 214     def install_chroot(self,options):
 215         # nothing to do
 216         return True
 217
 218     # xxx this would not work with hostname != localhost as mylc-init-vserver was extracted locally
 219     def install_vserver(self,options):
 220         # we need build dir for vtest-init-vserver
 221         if self.is_local():
 222             # a full path for the local calls
 223             build_dir=self.path+"/build"
 224         else:
 225             # use a standard name - will be relative to HOME
 226             build_dir="tests-system-build"
 227         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 228         if self.run_in_host(build_checkout) != 0:
 229             raise Exception,"Cannot checkout build dir"
 230         # the repo url is taken from myplc-url
 231         # with the last two steps (i386/myplc...) removed
 232         repo_url = options.myplc_url
 233         repo_url = os.path.dirname(repo_url)
 234         repo_url = os.path.dirname(repo_url)
 235         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 236             (build_dir,self.vservername,repo_url,self.vserverip)
 237         if self.run_in_host(create_vserver) != 0:
 238             raise Exception,"Could not create vserver for %s"%self.vservername
 239         return True
 240
 241     def install(self,options):
 242         if self.vserver:
 243             return self.install_vserver(options)
 244         else:
 245             return self.install_chroot(options)
 246
 247     ### install_rpm
 248     def install_rpm_chroot(self,options):
 249         utils.header('Installing from %s'%options.myplc_url)
 250         url=options.myplc_url
 251         self.run_in_host('rpm -Uvh '+url)
 252         self.run_in_host('service plc mount')
 253         return True
 254
 255     def install_rpm_vserver(self,options):
 256         self.run_in_guest("yum -y install myplc-native")
 257         return True
 258
 259     def install_rpm(self,options):
 260         if self.vserver:
 261             return self.install_rpm_vserver(options)
 262         else:
 263             return self.install_rpm_chroot(options)
 264
 265     ###
 266     def configure(self,options):
 267         tmpname='%s/%s.plc-config-tty'%(options.path,self.name())
 268         fileconf=open(tmpname,'w')
 269         for var in [ 'PLC_NAME',
 270                      'PLC_ROOT_PASSWORD',
 271                      'PLC_ROOT_USER',
 272                      'PLC_MAIL_ENABLED',
 273                      'PLC_MAIL_SUPPORT_ADDRESS',
 274                      'PLC_DB_HOST',
 275                      'PLC_API_HOST',
 276                      'PLC_WWW_HOST',
 277                      'PLC_BOOT_HOST',
 278                      'PLC_NET_DNS1',
 279                      'PLC_NET_DNS2']:
 280             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 281         fileconf.write('w\n')
 282         fileconf.write('q\n')
 283         fileconf.close()
 284         utils.system('cat %s'%tmpname)
 285         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 286         utils.system('rm %s'%tmpname)
 287         return True
 288
 289     # the chroot install is slightly different to this respect
 290     def start(self, options):
 291         if self.vserver:
 292             self.run_in_guest('service plc start')
 293         else:
 294             self.run_in_host('service plc start')
 295         return True
 296
 297     def stop(self, options):
 298         if self.vserver:
 299             self.run_in_guest('service plc stop')
 300         else:
 301             self.run_in_host('service plc stop')
 302         return True
 303
 304     # could use a TestKey class
 305     def store_keys(self, options):
 306         for key_spec in self.plc_spec['keys']:
 307             TestKey(self,key_spec).store_key()
 308         return True
 309
 310     def clean_keys(self, options):
 311         utils.system("rm -rf %s/keys/"%self.path)
 312
 313     def sites (self,options):
 314         return self.do_sites(options)
 315
 316     def clean_sites (self,options):
 317         return self.do_sites(options,action="delete")
 318
 319     def do_sites (self,options,action="add"):
 320         for site_spec in self.plc_spec['sites']:
 321             test_site = TestSite (self,site_spec)
 322             if (action != "add"):
 323                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 324                 test_site.delete_site()
 325                 # deleted with the site
 326                 #test_site.delete_users()
 327                 continue
 328             else:
 329                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 330                 test_site.create_site()
 331                 test_site.create_users()
 332         return True
 333
 334     def nodes (self, options):
 335         return self.do_nodes(options)
 336     def clean_nodes (self, options):
 337         return self.do_nodes(options,action="delete")
 338
 339     def do_nodes (self, options,action="add"):
 340         for site_spec in self.plc_spec['sites']:
 341             test_site = TestSite (self,site_spec)
 342             if action != "add":
 343                 utils.header("Deleting nodes in site %s"%test_site.name())
 344                 for node_spec in site_spec['nodes']:
 345                     test_node=TestNode(self,test_site,node_spec)
 346                     utils.header("Deleting %s"%test_node.name())
 347                     test_node.delete_node()
 348             else:
 349                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 350                 for node_spec in site_spec['nodes']:
 351                     utils.show_spec('Creating node %s'%node_spec,node_spec)
 352                     test_node = TestNode (self,test_site,node_spec)
 353                     test_node.create_node ()
 354         return True
 355
 356     # create nodegroups if needed, and populate
 357     # no need for a clean_nodegroups if we are careful enough
 358     def nodegroups (self, options):
 359         # 1st pass to scan contents
 360         groups_dict = {}
 361         for site_spec in self.plc_spec['sites']:
 362             test_site = TestSite (self,site_spec)
 363             for node_spec in site_spec['nodes']:
 364                 test_node=TestNode (self,test_site,node_spec)
 365                 if node_spec.has_key('nodegroups'):
 366                     nodegroupnames=node_spec['nodegroups']
 367                     if isinstance(nodegroupnames,StringTypes):
 368                         nodegroupnames = [ nodegroupnames ]
 369                     for nodegroupname in nodegroupnames:
 370                         if not groups_dict.has_key(nodegroupname):
 371                             groups_dict[nodegroupname]=[]
 372                         groups_dict[nodegroupname].append(test_node.name())
 373         auth=self.auth_root()
 374         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 375             try:
 376                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 377             except:
 378                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 379             for node in group_nodes:
 380                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 381         return True
 382
 383     def all_hostnames (self) :
 384         hostnames = []
 385         for site_spec in self.plc_spec['sites']:
 386             hostnames += [ node_spec['node_fields']['hostname'] \
 387                            for node_spec in site_spec['nodes'] ]
 388         return hostnames
 389
 390     # gracetime : during the first <gracetime> minutes nothing gets printed
 391     def do_nodes_booted (self, minutes, gracetime=2):
 392         # compute timeout
 393         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 394         graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
 395         # the nodes that haven't checked yet - start with a full list and shrink over time
 396         tocheck = self.all_hostnames()
 397         utils.header("checking nodes %r"%tocheck)
 398         # create a dict hostname -> status
 399         status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
 400         while tocheck:
 401             # get their status
 402             tocheck_status=self.server.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
 403             # update status
 404             for array in tocheck_status:
 405                 hostname=array['hostname']
 406                 boot_state=array['boot_state']
 407                 if boot_state == 'boot':
 408                     utils.header ("%s has reached the 'boot' state"%hostname)
 409                 else:
 410                     # if it's a real node, never mind
 411                     (site_spec,node_spec)=self.locate_node(hostname)
 412                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 413                         utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
 414                         # let's cheat
 415                         boot_state = 'boot'
 416                     if datetime.datetime.now() > graceout:
 417                         utils.header ("%s still in '%s' state"%(hostname,boot_state))
 418                         graceout=datetime.datetime.now()+datetime.timedelta(1)
 419                 status[hostname] = boot_state
 420             # refresh tocheck
 421             tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
 422             if not tocheck:
 423                 return True
 424             if datetime.datetime.now() > timeout:
 425                 for hostname in tocheck:
 426                     utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
 427                 return False
 428             # otherwise, sleep for a while
 429             time.sleep(15)
 430         # only useful in empty plcs
 431         return True
 432
 433     def nodes_booted(self,options):
 434         return self.do_nodes_booted(minutes=5)
 435
 436     #to scan and store the nodes's public keys and avoid to ask for confirmation when  ssh
 437     def scan_publicKeys(self,hostnames):
 438         try:
 439             temp_knownhosts="/root/known_hosts"
 440             remote_knownhosts="/root/.ssh/known_hosts"
 441             self.run_in_host("touch %s"%temp_knownhosts )
 442             for hostname in hostnames:
 443                 utils.header("Scan Public %s key and store it in the known_host file(under the root image) "%hostname)
 444                 scan=self.run_in_host('ssh-keyscan -t rsa %s >> %s '%(hostname,temp_knownhosts))
 445             #Store the public keys in the right root image
 446             self.copy_in_guest(temp_knownhosts,remote_knownhosts,True)
 447             #clean the temp keys file used
 448             self.run_in_host('rm -f  %s '%temp_knownhosts )
 449         except Exception, err:
 450             print err
 451
 452     def do_check_nodesSsh(self,minutes):
 453         # compute timeout
 454         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 455         #graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
 456         tocheck = self.all_hostnames()
 457         self.scan_publicKeys(tocheck)
 458         utils.header("checking Connectivity on nodes %r"%tocheck)
 459         while tocheck:
 460             for hostname in tocheck:
 461                 # try to ssh in nodes
 462                 access=self.run_in_guest('ssh -i /etc/planetlab/root_ssh_key.rsa root@%s date'%hostname )
 463                 if (not access):
 464                     utils.header('The node %s is sshable -->'%hostname)
 465                     # refresh tocheck
 466                     tocheck.remove(hostname)
 467                 else:
 468                     (site_spec,node_spec)=self.locate_node(hostname)
 469                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 470                         utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
 471                     tocheck.remove(hostname)
 472             if not tocheck:
 473                 return True
 474             if datetime.datetime.now() > timeout:
 475                 for hostname in tocheck:
 476                     utils.header("FAILURE to ssh into %s"%hostname)
 477                 return False
 478             # otherwise, sleep for a while
 479             time.sleep(15)
 480         # only useful in empty plcs
 481         return True
 482
 483     def nodes_ssh(self, options):
 484         return  self.do_check_nodesSsh(minutes=2)
 485
 486     def bootcd (self, options):
 487         for site_spec in self.plc_spec['sites']:
 488             test_site = TestSite (self,site_spec)
 489             for node_spec in site_spec['nodes']:
 490                 test_node=TestNode (self,test_site,node_spec)
 491                 test_node.create_boot_cd(options.path)
 492         return True
 493
 494     def initscripts (self, options):
 495         for initscript in self.plc_spec['initscripts']:
 496             utils.show_spec('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 497             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 498         return True
 499
 500     def slices (self, options):
 501         return self.do_slices()
 502
 503     def clean_slices (self, options):
 504         return self.do_slices("delete")
 505
 506     def do_slices (self,  action="add"):
 507         for slice in self.plc_spec['slices']:
 508             site_spec = self.locate_site (slice['sitename'])
 509             test_site = TestSite(self,site_spec)
 510             test_slice=TestSlice(self,test_site,slice)
 511             if action != "add":
 512                 utils.header("Deleting slices in site %s"%test_site.name())
 513                 test_slice.delete_slice()
 514             else:
 515                 utils.show_spec("Creating slice",slice)
 516                 test_slice.create_slice()
 517                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
 518         return True
 519
 520     def check_slices(self, options):
 521         for slice_spec in self.plc_spec['slices']:
 522             site_spec = self.locate_site (slice_spec['sitename'])
 523             test_site = TestSite(self,site_spec)
 524             test_slice=TestSlice(self,test_site,slice_spec)
 525             status=test_slice.do_check_slice(options)
 526             if (not status):
 527                 return False
 528         return status
 529
 530     def start_nodes (self, options):
 531         self.kill_all_qemus()
 532         utils.header("Starting  nodes")
 533         for site_spec in self.plc_spec['sites']:
 534             TestSite(self,site_spec).start_nodes (options)
 535         return True
 536
 537     def stop_nodes (self, options):
 538         self.kill_all_qemus()
 539         return True
 540
 541     # returns the filename to use for sql dump/restore, using options.dbname if set
 542     def dbfile (self, database, options):
 543         # uses options.dbname if it is found
 544         try:
 545             name=options.dbname
 546             if not isinstance(name,StringTypes):
 547                 raise Exception
 548         except:
 549             t=datetime.datetime.now()
 550             d=t.date()
 551             name=str(d)
 552         return "/root/%s-%s.sql"%(database,name)
 553
 554     def db_dump(self, options):
 555
 556         dump=self.dbfile("planetab4",options)
 557         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 558         utils.header('Dumped planetlab4 database in %s'%dump)
 559         return True
 560
 561     def db_restore(self, options):
 562         dump=self.dbfile("planetab4",options)
 563         ##stop httpd service
 564         self.run_in_guest('service httpd stop')
 565         # xxx - need another wrapper
 566         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 567         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 568         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 569         ##starting httpd service
 570         self.run_in_guest('service httpd start')
 571
 572         utils.header('Database restored from ' + dump)
 573
 574     @standby_generic
 575     def standby_1(): pass
 576     @standby_generic
 577     def standby_2(): pass
 578     @standby_generic
 579     def standby_3(): pass
 580     @standby_generic
 581     def standby_4(): pass
 582     @standby_generic
 583     def standby_5(): pass
 584     @standby_generic
 585     def standby_6(): pass
 586     @standby_generic
 587     def standby_7(): pass
 588     @standby_generic
 589     def standby_8(): pass
 590     @standby_generic
 591     def standby_9(): pass
 592     @standby_generic
 593     def standby_10(): pass
 594     @standby_generic
 595     def standby_11(): pass
 596     @standby_generic
 597     def standby_12(): pass
 598     @standby_generic
 599     def standby_13(): pass
 600     @standby_generic
 601     def standby_14(): pass
 602     @standby_generic
 603     def standby_15(): pass
 604     @standby_generic
 605     def standby_16(): pass
 606     @standby_generic
 607     def standby_17(): pass
 608     @standby_generic
 609     def standby_18(): pass
 610     @standby_generic
 611     def standby_19(): pass
 612     @standby_generic
 613     def standby_20(): pass
 614