system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16 from TestSlice import TestSlice
  17
  18 # inserts a backslash before each occurence of the following chars
  19 # \ " ' < > & | ; ( ) $ *
  20 def backslash_shell_specials (command):
  21     result=''
  22     for char in command:
  23         if char in "\\\"'<>&|;()$*":
  24             result +='\\'+char
  25         else:
  26             result +=char
  27     return result
  28
  29 # step methods must take (self, options) and return a boolean
  30
  31 class TestPlc:
  32
  33     def __init__ (self,plc_spec):
  34         self.plc_spec=plc_spec
  35         self.path=os.path.dirname(sys.argv[0])
  36         try:
  37             self.vserverip=plc_spec['vserverip']
  38             self.vservername=plc_spec['vservername']
  39             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  40             self.vserver=True
  41         except:
  42             self.vserver=False
  43             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  44         utils.header('Using API url %s'%self.url)
  45         self.server=xmlrpclib.Server(self.url,allow_none=True)
  46
  47     def name(self):
  48         name=self.plc_spec['name']
  49         if self.vserver:
  50             return name+"[%s]"%self.vservername
  51         else:
  52             return name+"[chroot]"
  53
  54     def is_local (self):
  55         return self.plc_spec['hostname'] == 'localhost'
  56
  57     # define the API methods on this object through xmlrpc
  58     # would help, but not strictly necessary
  59     def connect (self):
  60         pass
  61
  62     # command gets run in the chroot/vserver
  63     def host_to_guest(self,command):
  64         if self.vserver:
  65             return "vserver %s exec %s"%(self.vservername,command)
  66         else:
  67             return "chroot /plc/root %s"%backslash_shell_specials(command)
  68
  69     # command gets run on the right box
  70     def to_host(self,command):
  71         if self.is_local():
  72             return command
  73         else:
  74             return "ssh %s %s"%(self.plc_spec['hostname'],backslash_shell_specials(command))
  75
  76     def full_command(self,command):
  77         return self.to_host(self.host_to_guest(command))
  78
  79     def run_in_guest (self,command):
  80         return utils.system(self.full_command(command))
  81     def run_in_host (self,command):
  82         return utils.system(self.to_host(command))
  83
  84     # xxx quick n dirty
  85     def run_in_guest_piped (self,local,remote):
  86         return utils.system(local+" | "+self.full_command(remote))
  87
  88     # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
  89     def copy_in_guest (self, localfile, remotefile, in_data=False):
  90         if in_data:
  91             chroot_dest="/plc/data"
  92         else:
  93             chroot_dest="/plc/root"
  94         if self.is_local():
  95             if not self.vserver:
  96                 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
  97             else:
  98                 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
  99         else:
 100             if not self.vserver:
 101                 utils.system("scp %s %s:%s/%s"%(localfile,self.plc_spec['hostname'],chroot_dest,remotefile))
 102             else:
 103                 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.plc_spec['hostname'],self.vservername,remotefile))
 104
 105     def auth_root (self):
 106         return {'Username':self.plc_spec['PLC_ROOT_USER'],
 107                 'AuthMethod':'password',
 108                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
 109                 'Role' : self.plc_spec['role']
 110                 }
 111     def locate_site (self,sitename):
 112         for site in self.plc_spec['sites']:
 113             if site['site_fields']['name'] == sitename:
 114                 return site
 115             if site['site_fields']['login_base'] == sitename:
 116                 return site
 117         raise Exception,"Cannot locate site %s"%sitename
 118
 119     def locate_node (self,nodename):
 120         for site in self.plc_spec['sites']:
 121             for node in site['nodes']:
 122                 if node['node_fields']['hostname'] == nodename:
 123                     return (site,node)
 124         raise Exception,"Cannot locate node %s"%nodename
 125
 126     def locate_key (self,keyname):
 127         for key in self.plc_spec['keys']:
 128             if key['name'] == keyname:
 129                 return key
 130         raise Exception,"Cannot locate key %s"%keyname
 131
 132     # this should be run on the nodes' host_box, not locally to the plc
 133     def kill_all_vmwares(self):
 134         utils.header('Killing any running vmware or vmplayer instance')
 135         utils.system('pgrep vmware | xargs -r kill')
 136         utils.system('pgrep vmplayer | xargs -r kill ')
 137         utils.system('pgrep vmware | xargs -r kill -9')
 138         utils.system('pgrep vmplayer | xargs -r kill -9')
 139
 140     def kill_all_qemus(self):
 141         for site_spec in self.plc_spec['sites']:
 142             test_site = TestSite (self,site_spec)
 143             for node_spec in site_spec['nodes']:
 144                 TestNode (self,test_site,node_spec).stop_qemu()
 145
 146     def clear_ssh_config (self,options):
 147         # install local ssh_config file as root's .ssh/config - ssh should be quiet
 148         # dir might need creation first
 149         self.run_in_guest("mkdir /root/.ssh")
 150         self.run_in_guest("chmod 700 /root/.ssh")
 151         # this does not work - > redirection somehow makes it until an argument to cat
 152         #self.run_in_guest_piped("cat ssh_config","cat > /root/.ssh/config")
 153         self.copy_in_guest("ssh_config","/root/.ssh/config",True)
 154         return True
 155
 156     #################### step methods
 157
 158     ### uninstall
 159     def uninstall_chroot(self,options):
 160         self.run_in_host('service plc safestop')
 161         #####detecting the last myplc version installed and remove it
 162         self.run_in_host('rpm -e myplc')
 163         ##### Clean up the /plc directory
 164         self.run_in_host('rm -rf  /plc/data')
 165         ##### stop any running vservers
 166         self.run_in_host('for vserver in $(ls /vservers/* | sed -e s,/vservers/,,) ; do vserver $vserver stop ; done')
 167         return True
 168
 169     def uninstall_vserver(self,options):
 170         self.run_in_host("vserver --silent %s delete"%self.vservername)
 171         return True
 172
 173     def uninstall(self,options):
 174         # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
 175         # it sounds safer to have the former uninstalled too
 176         # now the vserver method cannot be invoked for chroot instances as vservername is required
 177         if self.vserver:
 178             self.uninstall_vserver(options)
 179             self.uninstall_chroot(options)
 180         else:
 181             self.uninstall_chroot(options)
 182         return True
 183
 184     ### install
 185     def install_chroot(self,options):
 186         # nothing to do
 187         return True
 188
 189     # xxx this would not work with hostname != localhost as mylc-init-vserver was extracted locally
 190     def install_vserver(self,options):
 191         # we need build dir for vtest-init-vserver
 192         if self.is_local():
 193             # a full path for the local calls
 194             build_dir=self.path+"/build"
 195         else:
 196             # use a standard name - will be relative to HOME
 197             build_dir="tests-system-build"
 198         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 199         if self.run_in_host(build_checkout) != 0:
 200             raise Exception,"Cannot checkout build dir"
 201         # the repo url is taken from myplc-url
 202         # with the last two steps (i386/myplc...) removed
 203         repo_url = options.myplc_url
 204         repo_url = os.path.dirname(repo_url)
 205         repo_url = os.path.dirname(repo_url)
 206         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 207             (build_dir,self.vservername,repo_url,self.vserverip)
 208         if self.run_in_host(create_vserver) != 0:
 209             raise Exception,"Could not create vserver for %s"%self.vservername
 210         return True
 211
 212     def install(self,options):
 213         if self.vserver:
 214             return self.install_vserver(options)
 215         else:
 216             return self.install_chroot(options)
 217
 218     ### install_rpm
 219     def install_rpm_chroot(self,options):
 220         utils.header('Installing from %s'%options.myplc_url)
 221         url=options.myplc_url
 222         self.run_in_host('rpm -Uvh '+url)
 223         self.run_in_host('service plc mount')
 224         return True
 225
 226     def install_rpm_vserver(self,options):
 227         self.run_in_guest("yum -y install myplc-native")
 228         return True
 229
 230     def install_rpm(self,options):
 231         if self.vserver:
 232             return self.install_rpm_vserver(options)
 233         else:
 234             return self.install_rpm_chroot(options)
 235
 236     ###
 237     def configure(self,options):
 238         tmpname='%s/%s.plc-config-tty'%(options.path,self.name())
 239         fileconf=open(tmpname,'w')
 240         for var in [ 'PLC_NAME',
 241                      'PLC_ROOT_PASSWORD',
 242                      'PLC_ROOT_USER',
 243                      'PLC_MAIL_ENABLED',
 244                      'PLC_MAIL_SUPPORT_ADDRESS',
 245                      'PLC_DB_HOST',
 246                      'PLC_API_HOST',
 247                      'PLC_WWW_HOST',
 248                      'PLC_BOOT_HOST',
 249                      'PLC_NET_DNS1',
 250                      'PLC_NET_DNS2']:
 251             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 252         fileconf.write('w\n')
 253         fileconf.write('q\n')
 254         fileconf.close()
 255         utils.system('cat %s'%tmpname)
 256         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 257         utils.system('rm %s'%tmpname)
 258         return True
 259
 260     # the chroot install is slightly different to this respect
 261     def start(self, options):
 262         if self.vserver:
 263             self.run_in_guest('service plc start')
 264         else:
 265             self.run_in_host('service plc start')
 266         return True
 267
 268     def stop(self, options):
 269         if self.vserver:
 270             self.run_in_guest('service plc stop')
 271         else:
 272             self.run_in_host('service plc stop')
 273         return True
 274
 275     # could use a TestKey class
 276     def store_keys(self, options):
 277         for key_spec in self.plc_spec['keys']:
 278             TestKey(self,key_spec).store_key()
 279         return True
 280
 281     def clean_keys(self, options):
 282         utils.system("rm -rf %s/keys/"%self.path)
 283
 284     def sites (self,options):
 285         return self.do_sites(options)
 286
 287     def clean_sites (self,options):
 288         return self.do_sites(options,action="delete")
 289
 290     def do_sites (self,options,action="add"):
 291         for site_spec in self.plc_spec['sites']:
 292             test_site = TestSite (self,site_spec)
 293             if (action != "add"):
 294                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 295                 test_site.delete_site()
 296                 # deleted with the site
 297                 #test_site.delete_users()
 298                 continue
 299             else:
 300                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 301                 test_site.create_site()
 302                 test_site.create_users()
 303         return True
 304
 305     def nodes (self, options):
 306         return self.do_nodes(options)
 307     def clean_nodes (self, options):
 308         return self.do_nodes(options,action="delete")
 309
 310     def do_nodes (self, options,action="add"):
 311         for site_spec in self.plc_spec['sites']:
 312             test_site = TestSite (self,site_spec)
 313             if action != "add":
 314                 utils.header("Deleting nodes in site %s"%test_site.name())
 315                 for node_spec in site_spec['nodes']:
 316                     test_node=TestNode(self,test_site,node_spec)
 317                     utils.header("Deleting %s"%test_node.name())
 318                     test_node.delete_node()
 319             else:
 320                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 321                 for node_spec in site_spec['nodes']:
 322                     utils.show_spec('Creating node %s'%node_spec,node_spec)
 323                     test_node = TestNode (self,test_site,node_spec)
 324                     test_node.create_node ()
 325         return True
 326
 327     # create nodegroups if needed, and populate
 328     # no need for a clean_nodegroups if we are careful enough
 329     def nodegroups (self, options):
 330         # 1st pass to scan contents
 331         groups_dict = {}
 332         for site_spec in self.plc_spec['sites']:
 333             test_site = TestSite (self,site_spec)
 334             for node_spec in site_spec['nodes']:
 335                 test_node=TestNode (self,test_site,node_spec)
 336                 if node_spec.has_key('nodegroups'):
 337                     nodegroupnames=node_spec['nodegroups']
 338                     if isinstance(nodegroupnames,StringTypes):
 339                         nodegroupnames = [ nodegroupnames ]
 340                     for nodegroupname in nodegroupnames:
 341                         if not groups_dict.has_key(nodegroupname):
 342                             groups_dict[nodegroupname]=[]
 343                         groups_dict[nodegroupname].append(test_node.name())
 344         auth=self.auth_root()
 345         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 346             try:
 347                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 348             except:
 349                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 350             for node in group_nodes:
 351                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 352         return True
 353
 354     def all_hostnames (self) :
 355         hostnames = []
 356         for site_spec in self.plc_spec['sites']:
 357             hostnames += [ node_spec['node_fields']['hostname'] \
 358                            for node_spec in site_spec['nodes'] ]
 359         return hostnames
 360
 361     # gracetime : during the first <gracetime> minutes nothing gets printed
 362     def do_check_nodesStatus (self, minutes, gracetime=2):
 363         # compute timeout
 364         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 365         graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
 366         # the nodes that haven't checked yet - start with a full list and shrink over time
 367         tocheck = self.all_hostnames()
 368         utils.header("checking nodes %r"%tocheck)
 369         # create a dict hostname -> status
 370         status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
 371         while tocheck:
 372             # get their status
 373             tocheck_status=self.server.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
 374             # update status
 375             for array in tocheck_status:
 376                 hostname=array['hostname']
 377                 boot_state=array['boot_state']
 378                 if boot_state == 'boot':
 379                     utils.header ("%s has reached the 'boot' state"%hostname)
 380                 else:
 381                     # if it's a real node, never mind
 382                     (site_spec,node_spec)=self.locate_node(hostname)
 383                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 384                         utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
 385                         # let's cheat
 386                         boot_state = 'boot'
 387                     if datetime.datetime.now() > graceout:
 388                         utils.header ("%s still in '%s' state"%(hostname,boot_state))
 389                         graceout=datetime.datetime.now()+datetime.timedelta(1)
 390                 status[hostname] = boot_state
 391             # refresh tocheck
 392             tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
 393             if not tocheck:
 394                 return True
 395             if datetime.datetime.now() > timeout:
 396                 for hostname in tocheck:
 397                     utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
 398                 return False
 399             # otherwise, sleep for a while
 400             time.sleep(15)
 401         # only useful in empty plcs
 402         return True
 403
 404     def check_nodesStatus(self,options):
 405         return self.do_check_nodesStatus(minutes=5)
 406
 407     #to scan and store the nodes's public keys and avoid to ask for confirmation when  ssh
 408     def scan_publicKeys(self,hostnames):
 409         try:
 410             temp_knownhosts="/root/known_hosts"
 411             remote_knownhosts="/root/.ssh/known_hosts"
 412             self.run_in_host("touch %s"%temp_knownhosts )
 413             for hostname in hostnames:
 414                 utils.header("Scan Public %s key and store it in the known_host file(under the root image) "%hostname)
 415                 scan=self.run_in_host('ssh-keyscan -t rsa %s >> %s '%(hostname,temp_knownhosts))
 416             #Store the public keys in the right root image
 417             self.copy_in_guest(temp_knownhosts,remote_knownhosts,True)
 418             #clean the temp keys file used
 419             self.run_in_host('rm -f  %s '%temp_knownhosts )
 420         except Exception, err:
 421             print err
 422
 423     def do_check_nodesSsh(self,minutes):
 424         # compute timeout
 425         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 426         #graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
 427         tocheck = self.all_hostnames()
 428         self.scan_publicKeys(tocheck)
 429         utils.header("checking Connectivity on nodes %r"%tocheck)
 430         while tocheck:
 431             for hostname in tocheck:
 432                 # try to ssh in nodes
 433                 access=self.run_in_guest('ssh -i /etc/planetlab/root_ssh_key.rsa root@%s date'%hostname )
 434                 if (not access):
 435                     utils.header('The node %s is sshable -->'%hostname)
 436                     # refresh tocheck
 437                     tocheck.remove(hostname)
 438                 else:
 439                     (site_spec,node_spec)=self.locate_node(hostname)
 440                     if TestNode.is_real_model(node_spec['node_fields']['model']):
 441                         utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
 442                     tocheck.remove(hostname)
 443             if not tocheck:
 444                 return True
 445             if datetime.datetime.now() > timeout:
 446                 for hostname in tocheck:
 447                     utils.header("FAILURE to ssh into %s"%hostname)
 448                 return False
 449             # otherwise, sleep for a while
 450             time.sleep(15)
 451         # only useful in empty plcs
 452         return True
 453
 454     def check_nodesConnectivity(self, options):
 455         return  self.do_check_nodesSsh(minutes=2)
 456
 457     def standby(self,options):
 458         #Method for waiting a while when nodes are booting and being sshable,giving time to NM to be up
 459         utils.header('Entering in StanbdBy mode at %s'%datetime.datetime.now())
 460         time.sleep(900)
 461         utils.header('Exist StandBy mode at %s'%datetime.datetime.now())
 462         return True
 463
 464     def bootcd (self, options):
 465         for site_spec in self.plc_spec['sites']:
 466             test_site = TestSite (self,site_spec)
 467             for node_spec in site_spec['nodes']:
 468                 test_node=TestNode (self,test_site,node_spec)
 469                 test_node.create_boot_cd(options.path)
 470         return True
 471
 472     def initscripts (self, options):
 473         for initscript in self.plc_spec['initscripts']:
 474             utils.show_spec('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 475             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 476         return True
 477
 478     def slices (self, options):
 479         return self.do_slices()
 480
 481     def clean_slices (self, options):
 482         return self.do_slices("delete")
 483
 484     def do_slices (self,  action="add"):
 485         for slice in self.plc_spec['slices']:
 486             site_spec = self.locate_site (slice['sitename'])
 487             test_site = TestSite(self,site_spec)
 488             test_slice=TestSlice(self,test_site,slice)
 489             if action != "add":
 490                 utils.header("Deleting slices in site %s"%test_site.name())
 491                 test_slice.delete_slice()
 492             else:
 493                 utils.show_spec("Creating slice",slice)
 494                 test_slice.create_slice()
 495                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
 496         return True
 497
 498     def check_slices(self, options):
 499         for slice_spec in self.plc_spec['slices']:
 500             site_spec = self.locate_site (slice_spec['sitename'])
 501             test_site = TestSite(self,site_spec)
 502             test_slice=TestSlice(self,test_site,slice_spec)
 503             status=test_slice.do_check_slice(options)
 504             return status
 505
 506     def start_nodes (self, options):
 507         self.kill_all_vmwares()
 508         self.kill_all_qemus()
 509         utils.header("Starting vmware nodes")
 510         for site_spec in self.plc_spec['sites']:
 511             TestSite(self,site_spec).start_nodes (options)
 512         return True
 513
 514     def stop_nodes (self, options):
 515         self.kill_all_vmwares ()
 516         self.kill_all_qemus()
 517         return True
 518
 519     # returns the filename to use for sql dump/restore, using options.dbname if set
 520     def dbfile (self, database, options):
 521         # uses options.dbname if it is found
 522         try:
 523             name=options.dbname
 524             if not isinstance(name,StringTypes):
 525                 raise Exception
 526         except:
 527             t=datetime.datetime.now()
 528             d=t.date()
 529             name=str(d)
 530         return "/root/%s-%s.sql"%(database,name)
 531
 532     def db_dump(self, options):
 533
 534         dump=self.dbfile("planetab4",options)
 535         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 536         utils.header('Dumped planetlab4 database in %s'%dump)
 537         return True
 538
 539     def db_restore(self, options):
 540         dump=self.dbfile("planetab4",options)
 541         ##stop httpd service
 542         self.run_in_guest('service httpd stop')
 543         # xxx - need another wrapper
 544         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 545         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 546         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 547         ##starting httpd service
 548         self.run_in_guest('service httpd start')
 549
 550         utils.header('Database restored from ' + dump)