system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16 from TestSlice import TestSlice
  17
  18 # inserts a backslash before each occurence of the following chars
  19 # \ " ' < > & | ; ( ) $ * ~ @
  20 def backslash_shell_specials (command):
  21     result=''
  22     for char in command:
  23         if char in "\\\"'<>&|;()$*~@":
  24             result +='\\'+char
  25         else:
  26             result +=char
  27     return result
  28
  29 # step methods must take (self, options) and return a boolean
  30
  31 class TestPlc:
  32
  33     def __init__ (self,plc_spec):
  34         self.plc_spec=plc_spec
  35         self.path=os.path.dirname(sys.argv[0])
  36         try:
  37             self.vserverip=plc_spec['vserverip']
  38             self.vservername=plc_spec['vservername']
  39             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  40             self.vserver=True
  41         except:
  42             self.vserver=False
  43             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  44         utils.header('Using API url %s'%self.url)
  45         self.server=xmlrpclib.Server(self.url,allow_none=True)
  46
  47     def name(self):
  48         name=self.plc_spec['name']
  49         if self.vserver:
  50             return name+"[%s]"%self.vservername
  51         else:
  52             return name+"[chroot]"
  53
  54     def is_local (self):
  55         return self.plc_spec['hostname'] == 'localhost'
  56
  57     # define the API methods on this object through xmlrpc
  58     # would help, but not strictly necessary
  59     def connect (self):
  60         pass
  61
  62     # command gets run in the chroot/vserver
  63     def host_to_guest(self,command):
  64         if self.vserver:
  65             return "vserver %s exec %s"%(self.vservername,command)
  66         else:
  67             return "chroot /plc/root %s"%backslash_shell_specials(command)
  68
  69     # command gets run on the right box
  70     def to_host(self,command):
  71         if self.is_local():
  72             return command
  73         else:
  74             return "ssh %s %s"%(self.plc_spec['hostname'],backslash_shell_specials(command))
  75
  76     def full_command(self,command):
  77         return self.to_host(self.host_to_guest(command))
  78
  79     def run_in_guest (self,command):
  80         return utils.system(self.full_command(command))
  81     def run_in_host (self,command):
  82         return utils.system(self.to_host(command))
  83
  84     # xxx quick n dirty
  85     def run_in_guest_piped (self,local,remote):
  86         return utils.system(local+" | "+self.full_command(remote))
  87
  88     # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
  89     def copy_in_guest (self, localfile, remotefile, in_data=False):
  90         if in_data:
  91             chroot_dest="/plc/data"
  92         else:
  93             chroot_dest="/plc/root"
  94         if self.is_local():
  95             if not self.vserver:
  96                 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
  97             else:
  98                 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
  99         else:
 100             if not self.vserver:
 101                 utils.system("scp %s %s:%s/%s"%(localfile,self.plc_spec['hostname'],chroot_dest,remotefile))
 102             else:
 103                 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.plc_spec['hostname'],self.vservername,remotefile))
 104
 105     def auth_root (self):
 106         return {'Username':self.plc_spec['PLC_ROOT_USER'],
 107                 'AuthMethod':'password',
 108                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
 109                 'Role' : self.plc_spec['role']
 110                 }
 111     def locate_site (self,sitename):
 112         for site in self.plc_spec['sites']:
 113             if site['site_fields']['name'] == sitename:
 114                 return site
 115             if site['site_fields']['login_base'] == sitename:
 116                 return site
 117         raise Exception,"Cannot locate site %s"%sitename
 118
 119     def locate_node (self,nodename):
 120         for site in self.plc_spec['sites']:
 121             for node in site['nodes']:
 122                 if node['node_fields']['hostname'] == nodename:
 123                     return (site,node)
 124         raise Exception,"Cannot locate node %s"%nodename
 125
 126     def locate_key (self,keyname):
 127         for key in self.plc_spec['keys']:
 128             if key['name'] == keyname:
 129                 return key
 130         raise Exception,"Cannot locate key %s"%keyname
 131
 132     #this to catch up all different hostboxes used in this plc
 133     def locate_hostBoxes(self,site_spec):
 134         #Get The first host box to avoid returning a long list with the same host box
 135         #in case  only one is used for all the nodes
 136         HostBoxes=[site_spec['nodes'][0]['host_box']]
 137         for node_spec in site_spec['nodes']:
 138             if node_spec['host_box']!= HostBoxes[0]:
 139                 HostBoxes.append( node_spec['host_box'])
 140
 141         return HostBoxes
 142
 143     def kill_all_qemus(self):
 144         for site_spec in self.plc_spec['sites']:
 145             test_site = TestSite (self,site_spec)
 146             hostboxes_list=self.locate_hostBoxes(site_spec)
 147             if (hostboxes_list):
 148                 for node_spec in site_spec['nodes']:
 149                     TestNode(self,test_site,node_spec).stop_qemu(node_spec)
 150             else:
 151                 utils.header("No emulated node running on this PLC config ignore the kill() step")
 152
 153     def clear_ssh_config (self,options):
 154         # install local ssh_config file as root's .ssh/config - ssh should be quiet
 155         # dir might need creation first
 156         self.run_in_guest("mkdir /root/.ssh")
 157         self.run_in_guest("chmod 700 /root/.ssh")
 158         # this does not work - > redirection somehow makes it until an argument to cat
 159         #self.run_in_guest_piped("cat ssh_config","cat > /root/.ssh/config")
 160         self.copy_in_guest("ssh_config","/root/.ssh/config",True)
 161         return True
 162
 163     #################### step methods
 164
 165     ### uninstall
 166     def uninstall_chroot(self,options):
 167         self.run_in_host('service plc safestop')
 168         #####detecting the last myplc version installed and remove it
 169         self.run_in_host('rpm -e myplc')
 170         ##### Clean up the /plc directory
 171         self.run_in_host('rm -rf  /plc/data')
 172         ##### stop any running vservers
 173         self.run_in_host('for vserver in $(ls /vservers/* | sed -e s,/vservers/,,) ; do vserver $vserver stop ; done')
 174         return True
 175
 176     def uninstall_vserver(self,options):
 177         self.run_in_host("vserver --silent %s delete"%self.vservername)
 178         return True
 179
 180     def uninstall(self,options):
 181         # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
 182         # it sounds safer to have the former uninstalled too
 183         # now the vserver method cannot be invoked for chroot instances as vservername is required
 184         if self.vserver:
 185             self.uninstall_vserver(options)
 186             self.uninstall_chroot(options)
 187         else:
 188             self.uninstall_chroot(options)
 189         return True
 190
 191     ### install
 192     def install_chroot(self,options):
 193         # nothing to do
 194         return True
 195
 196     # xxx this would not work with hostname != localhost as mylc-init-vserver was extracted locally
 197     def install_vserver(self,options):
 198         # we need build dir for vtest-init-vserver
 199         if self.is_local():
 200             # a full path for the local calls
 201             build_dir=self.path+"/build"
 202         else:
 203             # use a standard name - will be relative to HOME
 204             build_dir="tests-system-build"
 205         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 206         if self.run_in_host(build_checkout) != 0:
 207             raise Exception,"Cannot checkout build dir"
 208         # the repo url is taken from myplc-url
 209         # with the last two steps (i386/myplc...) removed
 210         repo_url = options.myplc_url
 211         repo_url = os.path.dirname(repo_url)
 212         repo_url = os.path.dirname(repo_url)
 213         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 214             (build_dir,self.vservername,repo_url,self.vserverip)
 215         if self.run_in_host(create_vserver) != 0:
 216             raise Exception,"Could not create vserver for %s"%self.vservername
 217         return True
 218
 219     def install(self,options):
 220         if self.vserver:
 221             return self.install_vserver(options)
 222         else:
 223             return self.install_chroot(options)
 224
 225     ### install_rpm
 226     def install_rpm_chroot(self,options):
 227         utils.header('Installing from %s'%options.myplc_url)
 228         url=options.myplc_url
 229         self.run_in_host('rpm -Uvh '+url)
 230         self.run_in_host('service plc mount')
 231         return True
 232
 233     def install_rpm_vserver(self,options):
 234         self.run_in_guest("yum -y install myplc-native")
 235         return True
 236
 237     def install_rpm(self,options):
 238         if self.vserver:
 239             return self.install_rpm_vserver(options)
 240         else:
 241             return self.install_rpm_chroot(options)
 242
 243     ###
 244     def configure(self,options):
 245         tmpname='%s/%s.plc-config-tty'%(options.path,self.name())
 246         fileconf=open(tmpname,'w')
 247         for var in [ 'PLC_NAME',
 248                      'PLC_ROOT_PASSWORD',
 249                      'PLC_ROOT_USER',
 250                      'PLC_MAIL_ENABLED',
 251                      'PLC_MAIL_SUPPORT_ADDRESS',
 252                      'PLC_DB_HOST',
 253                      'PLC_API_HOST',
 254                      'PLC_WWW_HOST',
 255                      'PLC_BOOT_HOST',
 256                      'PLC_NET_DNS1',
 257                      'PLC_NET_DNS2']:
 258             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 259         fileconf.write('w\n')
 260         fileconf.write('q\n')
 261         fileconf.close()
 262         utils.system('cat %s'%tmpname)
 263         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 264         utils.system('rm %s'%tmpname)
 265         return True
 266
 267     # the chroot install is slightly different to this respect
 268     def start(self, options):
 269         if self.vserver:
 270             self.run_in_guest('service plc start')
 271         else:
 272             self.run_in_host('service plc start')
 273         return True
 274
 275     def stop(self, options):
 276         if self.vserver:
 277             self.run_in_guest('service plc stop')
 278         else:
 279             self.run_in_host('service plc stop')
 280         return True
 281
 282     # could use a TestKey class
 283     def store_keys(self, options):
 284         for key_spec in self.plc_spec['keys']:
 285             TestKey(self,key_spec).store_key()
 286         return True
 287
 288     def clean_keys(self, options):
 289         utils.system("rm -rf %s/keys/"%self.path)
 290
 291     def sites (self,options):
 292         return self.do_sites(options)
 293
 294     def clean_sites (self,options):
 295         return self.do_sites(options,action="delete")
 296
 297     def do_sites (self,options,action="add"):
 298         for site_spec in self.plc_spec['sites']:
 299             test_site = TestSite (self,site_spec)
 300             if (action != "add"):
 301                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 302                 test_site.delete_site()
 303                 # deleted with the site
 304                 #test_site.delete_users()
 305                 continue
 306             else:
 307                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 308                 test_site.create_site()
 309                 test_site.create_users()
 310         return True
 311
 312     def nodes (self, options):
 313         return self.do_nodes(options)
 314     def clean_nodes (self, options):
 315         return self.do_nodes(options,action="delete")
 316
 317     def do_nodes (self, options,action="add"):
 318         for site_spec in self.plc_spec['sites']:
 319             test_site = TestSite (self,site_spec)
 320             if action != "add":
 321                 utils.header("Deleting nodes in site %s"%test_site.name())
 322                 for node_spec in site_spec['nodes']:
 323                     test_node=TestNode(self,test_site,node_spec)
 324                     utils.header("Deleting %s"%test_node.name())
 325                     test_node.delete_node()
 326             else:
 327                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 328                 for node_spec in site_spec['nodes']:
 329                     utils.show_spec('Creating node %s'%node_spec,node_spec)
 330                     test_node = TestNode (self,test_site,node_spec)
 331                     test_node.create_node ()
 332         return True
 333
 334     # create nodegroups if needed, and populate
 335     # no need for a clean_nodegroups if we are careful enough
 336     def nodegroups (self, options):
 337         # 1st pass to scan contents
 338         groups_dict = {}
 339         for site_spec in self.plc_spec['sites']:
 340             test_site = TestSite (self,site_spec)
 341             for node_spec in site_spec['nodes']:
 342                 test_node=TestNode (self,test_site,node_spec)
 343                 if node_spec.has_key('nodegroups'):
 344                     nodegroupnames=node_spec['nodegroups']
 345                     if isinstance(nodegroupnames,StringTypes):
 346                         nodegroupnames = [ nodegroupnames ]
 347                     for nodegroupname in nodegroupnames:
 348                         if not groups_dict.has_key(nodegroupname):
 349                             groups_dict[nodegroupname]=[]
 350                         groups_dict[nodegroupname].append(test_node.name())
 351         auth=self.auth_root()
 352         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 353             try:
 354                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 355             except:
 356                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 357             for node in group_nodes:
 358                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 359         return True
 360
 361     def all_hostnames (self) :
 362         hostnames = []
 363         for site_spec in self.plc_spec['sites']:
 364             hostnames += [ node_spec['node_fields']['hostname'] \
 365                            for node_spec in site_spec['nodes'] ]
 366         return hostnames
 367
 368     # gracetime : during the first <gracetime> minutes nothing gets printed
 369     def do_check_nodesStatus (self, minutes, gracetime=2):
 370         # compute timeout
 371         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 372         graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
 373         # the nodes that haven't checked yet - start with a full list and shrink over time
 374         tocheck = self.all_hostnames()
 375         utils.header("checking nodes %r"%tocheck)
 376         # create a dict hostname -> status
 377         status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
 378         while tocheck:
 379             # get their status
 380             tocheck_status=self.server.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
 381             # update status
 382             for array in tocheck_status:
 383                 hostname=array['hostname']
 384                 boot_state=array['boot_state']
 385                 if boot_state == 'boot':
 386                     utils.header ("%s has reached the 'boot' state"%hostname)
 387                 else:
 388                     # if it's a real node, never mind
 389                     (site_spec,node_spec)=self.locate_node(hostname)
 390                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 391                         utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
 392                         # let's cheat
 393                         boot_state = 'boot'
 394                     if datetime.datetime.now() > graceout:
 395                         utils.header ("%s still in '%s' state"%(hostname,boot_state))
 396                         graceout=datetime.datetime.now()+datetime.timedelta(1)
 397                 status[hostname] = boot_state
 398             # refresh tocheck
 399             tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
 400             if not tocheck:
 401                 return True
 402             if datetime.datetime.now() > timeout:
 403                 for hostname in tocheck:
 404                     utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
 405                 return False
 406             # otherwise, sleep for a while
 407             time.sleep(15)
 408         # only useful in empty plcs
 409         return True
 410
 411     def check_nodesStatus(self,options):
 412         return self.do_check_nodesStatus(minutes=5)
 413
 414     #to scan and store the nodes's public keys and avoid to ask for confirmation when  ssh
 415     def scan_publicKeys(self,hostnames):
 416         try:
 417             temp_knownhosts="/root/known_hosts"
 418             remote_knownhosts="/root/.ssh/known_hosts"
 419             self.run_in_host("touch %s"%temp_knownhosts )
 420             for hostname in hostnames:
 421                 utils.header("Scan Public %s key and store it in the known_host file(under the root image) "%hostname)
 422                 scan=self.run_in_host('ssh-keyscan -t rsa %s >> %s '%(hostname,temp_knownhosts))
 423             #Store the public keys in the right root image
 424             self.copy_in_guest(temp_knownhosts,remote_knownhosts,True)
 425             #clean the temp keys file used
 426             self.run_in_host('rm -f  %s '%temp_knownhosts )
 427         except Exception, err:
 428             print err
 429
 430     def do_check_nodesSsh(self,minutes):
 431         # compute timeout
 432         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 433         #graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
 434         tocheck = self.all_hostnames()
 435         self.scan_publicKeys(tocheck)
 436         utils.header("checking Connectivity on nodes %r"%tocheck)
 437         while tocheck:
 438             for hostname in tocheck:
 439                 # try to ssh in nodes
 440                 access=self.run_in_guest('ssh -i /etc/planetlab/root_ssh_key.rsa root@%s date'%hostname )
 441                 if (not access):
 442                     utils.header('The node %s is sshable -->'%hostname)
 443                     # refresh tocheck
 444                     tocheck.remove(hostname)
 445                 else:
 446                     (site_spec,node_spec)=self.locate_node(hostname)
 447                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 448                         utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
 449                     tocheck.remove(hostname)
 450             if not tocheck:
 451                 return True
 452             if datetime.datetime.now() > timeout:
 453                 for hostname in tocheck:
 454                     utils.header("FAILURE to ssh into %s"%hostname)
 455                 return False
 456             # otherwise, sleep for a while
 457             time.sleep(15)
 458         # only useful in empty plcs
 459         return True
 460
 461     def check_nodesConnectivity(self, options):
 462         return  self.do_check_nodesSsh(minutes=2)
 463
 464     def standby(self,options):
 465         #Method for waiting a while when nodes are booting and being sshable,giving time to NM to be up
 466         utils.header('Entering in StanbdBy mode for 10min at %s'%datetime.datetime.now())
 467         time.sleep(600)
 468         utils.header('Exist StandBy mode at %s'%datetime.datetime.now())
 469         return True
 470
 471     def bootcd (self, options):
 472         for site_spec in self.plc_spec['sites']:
 473             test_site = TestSite (self,site_spec)
 474             for node_spec in site_spec['nodes']:
 475                 test_node=TestNode (self,test_site,node_spec)
 476                 test_node.create_boot_cd(options.path)
 477         return True
 478
 479     def initscripts (self, options):
 480         for initscript in self.plc_spec['initscripts']:
 481             utils.show_spec('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 482             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 483         return True
 484
 485     def slices (self, options):
 486         return self.do_slices()
 487
 488     def clean_slices (self, options):
 489         return self.do_slices("delete")
 490
 491     def do_slices (self,  action="add"):
 492         for slice in self.plc_spec['slices']:
 493             site_spec = self.locate_site (slice['sitename'])
 494             test_site = TestSite(self,site_spec)
 495             test_slice=TestSlice(self,test_site,slice)
 496             if action != "add":
 497                 utils.header("Deleting slices in site %s"%test_site.name())
 498                 test_slice.delete_slice()
 499             else:
 500                 utils.show_spec("Creating slice",slice)
 501                 test_slice.create_slice()
 502                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
 503         return True
 504
 505     def check_slices(self, options):
 506         for slice_spec in self.plc_spec['slices']:
 507             site_spec = self.locate_site (slice_spec['sitename'])
 508             test_site = TestSite(self,site_spec)
 509             test_slice=TestSlice(self,test_site,slice_spec)
 510             status=test_slice.do_check_slice(options)
 511             return status
 512
 513     def start_nodes (self, options):
 514         self.kill_all_qemus()
 515         utils.header("Starting  nodes")
 516         for site_spec in self.plc_spec['sites']:
 517             TestSite(self,site_spec).start_nodes (options)
 518         return True
 519
 520     def stop_nodes (self, options):
 521         self.kill_all_qemus()
 522         return True
 523
 524     # returns the filename to use for sql dump/restore, using options.dbname if set
 525     def dbfile (self, database, options):
 526         # uses options.dbname if it is found
 527         try:
 528             name=options.dbname
 529             if not isinstance(name,StringTypes):
 530                 raise Exception
 531         except:
 532             t=datetime.datetime.now()
 533             d=t.date()
 534             name=str(d)
 535         return "/root/%s-%s.sql"%(database,name)
 536
 537     def db_dump(self, options):
 538
 539         dump=self.dbfile("planetab4",options)
 540         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 541         utils.header('Dumped planetlab4 database in %s'%dump)
 542         return True
 543
 544     def db_restore(self, options):
 545         dump=self.dbfile("planetab4",options)
 546         ##stop httpd service
 547         self.run_in_guest('service httpd stop')
 548         # xxx - need another wrapper
 549         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 550         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 551         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 552         ##starting httpd service
 553         self.run_in_guest('service httpd start')
 554
 555         utils.header('Database restored from ' + dump)