system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16 from TestSlice import TestSlice
  17
  18 # inserts a backslash before each occurence of the following chars
  19 # \ " ' < > & | ; ( ) $ *
  20 def backslash_shell_specials (command):
  21     result=''
  22     for char in command:
  23         if char in "\\\"'<>&|;()$*":
  24             result +='\\'+char
  25         else:
  26             result +=char
  27     return result
  28
  29 # step methods must take (self, options) and return a boolean
  30
  31 class TestPlc:
  32
  33     def __init__ (self,plc_spec):
  34         self.plc_spec=plc_spec
  35         self.path=os.path.dirname(sys.argv[0])
  36         try:
  37             self.vserverip=plc_spec['vserverip']
  38             self.vservername=plc_spec['vservername']
  39             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  40             self.vserver=True
  41         except:
  42             self.vserver=False
  43             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  44         utils.header('Using API url %s'%self.url)
  45         self.server=xmlrpclib.Server(self.url,allow_none=True)
  46
  47     def name(self):
  48         name=self.plc_spec['name']
  49         if self.vserver:
  50             return name+"[%s]"%self.vservername
  51         else:
  52             return name+"[chroot]"
  53
  54     def is_local (self):
  55         return self.plc_spec['hostname'] == 'localhost'
  56
  57     # define the API methods on this object through xmlrpc
  58     # would help, but not strictly necessary
  59     def connect (self):
  60         pass
  61
  62     # command gets run in the chroot/vserver
  63     def host_to_guest(self,command):
  64         if self.vserver:
  65             return "vserver %s exec %s"%(self.vservername,command)
  66         else:
  67             return "chroot /plc/root %s"%backslash_shell_specials(command)
  68
  69     # command gets run on the right box
  70     def to_host(self,command):
  71         if self.is_local():
  72             return command
  73         else:
  74             return "ssh %s %s"%(self.plc_spec['hostname'],backslash_shell_specials(command))
  75
  76     def full_command(self,command):
  77         return self.to_host(self.host_to_guest(command))
  78
  79     def run_in_guest (self,command):
  80         return utils.system(self.full_command(command))
  81     def run_in_host (self,command):
  82         return utils.system(self.to_host(command))
  83
  84     # xxx quick n dirty
  85     def run_in_guest_piped (self,local,remote):
  86         return utils.system(local+" | "+self.full_command(remote))
  87
  88     # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
  89     def copy_in_guest (self, localfile, remotefile, in_data=False):
  90         if in_data:
  91             chroot_dest="/plc/data"
  92         else:
  93             chroot_dest="/plc/root"
  94         if self.is_local():
  95             if not self.vserver:
  96                 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
  97             else:
  98                 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
  99         else:
 100             if not self.vserver:
 101                 utils.system("scp %s %s:%s/%s"%(localfile,self.plc_spec['hostname'],chroot_dest,remotefile))
 102             else:
 103                 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.plc_spec['hostname'],self.vservername,remotefile))
 104
 105     def auth_root (self):
 106         return {'Username':self.plc_spec['PLC_ROOT_USER'],
 107                 'AuthMethod':'password',
 108                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
 109                 'Role' : self.plc_spec['role']
 110                 }
 111     def locate_site (self,sitename):
 112         for site in self.plc_spec['sites']:
 113             if site['site_fields']['name'] == sitename:
 114                 return site
 115             if site['site_fields']['login_base'] == sitename:
 116                 return site
 117         raise Exception,"Cannot locate site %s"%sitename
 118
 119     def locate_node (self,nodename):
 120         for site in self.plc_spec['sites']:
 121             for node in site['nodes']:
 122                 if node['node_fields']['hostname'] == nodename:
 123                     return (site,node)
 124         raise Exception,"Cannot locate node %s"%nodename
 125
 126     def locate_key (self,keyname):
 127         for key in self.plc_spec['keys']:
 128             if key['name'] == keyname:
 129                 return key
 130         raise Exception,"Cannot locate key %s"%keyname
 131
 132     # this should be run on the nodes' host_box, not locally to the plc
 133     def kill_all_vmwares(self):
 134         utils.header('Killing any running vmware or vmplayer instance')
 135         utils.system('pgrep vmware | xargs -r kill')
 136         utils.system('pgrep vmplayer | xargs -r kill ')
 137         utils.system('pgrep vmware | xargs -r kill -9')
 138         utils.system('pgrep vmplayer | xargs -r kill -9')
 139
 140     def kill_all_qemus(self):
 141         for site_spec in self.plc_spec['sites']:
 142             test_site = TestSite (self,site_spec)
 143             for node_spec in site_spec['nodes']:
 144                 TestNode (self,test_site,node_spec).stop_qemu()
 145
 146     def clear_ssh_config (self,options):
 147         # install local ssh_config file as root's .ssh/config - ssh should be quiet
 148         # dir might need creation first
 149         self.run_in_guest("mkdir /root/.ssh")
 150         self.run_in_guest("chmod 700 /root/.ssh")
 151         # this does not work - > redirection somehow makes it until an argument to cat
 152         #self.run_in_guest_piped("cat ssh_config","cat > /root/.ssh/config")
 153         self.copy_in_guest("ssh_config","/root/.ssh/config",True)
 154         return True
 155
 156     #################### step methods
 157
 158     ### uninstall
 159     def uninstall_chroot(self,options):
 160         self.run_in_host('service plc safestop')
 161         #####detecting the last myplc version installed and remove it
 162         self.run_in_host('rpm -e myplc')
 163         ##### Clean up the /plc directory
 164         self.run_in_host('rm -rf  /plc/data')
 165         ##### stop any running vservers
 166         self.run_in_host('for vserver in $(ls /vservers/* | sed -e s,/vservers/,,) ; do vserver $vserver stop ; done')
 167         return True
 168
 169     def uninstall_vserver(self,options):
 170         self.run_in_host("vserver --silent %s delete"%self.vservername)
 171         return True
 172
 173     def uninstall(self,options):
 174         # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
 175         # it sounds safer to have the former uninstalled too
 176         # now the vserver method cannot be invoked for chroot instances as vservername is required
 177         if self.vserver:
 178             self.uninstall_vserver(options)
 179             self.uninstall_chroot(options)
 180         else:
 181             self.uninstall_chroot(options)
 182         return True
 183
 184     ### install
 185     def install_chroot(self,options):
 186         # nothing to do
 187         return True
 188
 189     # xxx this would not work with hostname != localhost as mylc-init-vserver was extracted locally
 190     def install_vserver(self,options):
 191         # we need build dir for vtest-init-vserver
 192         if self.is_local():
 193             # a full path for the local calls
 194             build_dir=self.path+"/build"
 195         else:
 196             # use a standard name - will be relative to HOME
 197             build_dir="tests-system-build"
 198         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 199         if self.run_in_host(build_checkout) != 0:
 200             raise Exception,"Cannot checkout build dir"
 201         # the repo url is taken from myplc-url
 202         # with the last two steps (i386/myplc...) removed
 203         repo_url = options.myplc_url
 204         repo_url = os.path.dirname(repo_url)
 205         repo_url = os.path.dirname(repo_url)
 206         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 207             (build_dir,self.vservername,repo_url,self.vserverip)
 208         if self.run_in_host(create_vserver) != 0:
 209             raise Exception,"Could not create vserver for %s"%self.vservername
 210         return True
 211
 212     def install(self,options):
 213         if self.vserver:
 214             return self.install_vserver(options)
 215         else:
 216             return self.install_chroot(options)
 217
 218     ### install_rpm
 219     def install_rpm_chroot(self,options):
 220         utils.header('Installing from %s'%options.myplc_url)
 221         url=options.myplc_url
 222         self.run_in_host('rpm -Uvh '+url)
 223         self.run_in_host('service plc mount')
 224         return True
 225
 226     def install_rpm_vserver(self,options):
 227         self.run_in_guest("yum -y install myplc-native")
 228         return True
 229
 230     def install_rpm(self,options):
 231         if self.vserver:
 232             return self.install_rpm_vserver(options)
 233         else:
 234             return self.install_rpm_chroot(options)
 235
 236     ###
 237     def configure(self,options):
 238         tmpname='%s/%s.plc-config-tty'%(options.path,self.name())
 239         fileconf=open(tmpname,'w')
 240         for var in [ 'PLC_NAME',
 241                      'PLC_ROOT_PASSWORD',
 242                      'PLC_ROOT_USER',
 243                      'PLC_MAIL_ENABLED',
 244                      'PLC_MAIL_SUPPORT_ADDRESS',
 245                      'PLC_DB_HOST',
 246                      'PLC_API_HOST',
 247                      'PLC_WWW_HOST',
 248                      'PLC_BOOT_HOST',
 249                      'PLC_NET_DNS1',
 250                      'PLC_NET_DNS2']:
 251             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 252         fileconf.write('w\n')
 253         fileconf.write('q\n')
 254         fileconf.close()
 255         utils.system('cat %s'%tmpname)
 256         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 257         utils.system('rm %s'%tmpname)
 258         return True
 259
 260     # the chroot install is slightly different to this respect
 261     def start(self, options):
 262         if self.vserver:
 263             self.run_in_guest('service plc start')
 264         else:
 265             self.run_in_host('service plc start')
 266         return True
 267
 268     def stop(self, options):
 269         if self.vserver:
 270             self.run_in_guest('service plc stop')
 271         else:
 272             self.run_in_host('service plc stop')
 273         return True
 274
 275     # could use a TestKey class
 276     def store_keys(self, options):
 277         for key_spec in self.plc_spec['keys']:
 278             TestKey(self,key_spec).store_key()
 279         return True
 280
 281     def clean_keys(self, options):
 282         utils.system("rm -rf %s/keys/"%self.path)
 283
 284     def sites (self,options):
 285         return self.do_sites(options)
 286
 287     def clean_sites (self,options):
 288         return self.do_sites(options,action="delete")
 289
 290     def do_sites (self,options,action="add"):
 291         for site_spec in self.plc_spec['sites']:
 292             test_site = TestSite (self,site_spec)
 293             if (action != "add"):
 294                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 295                 test_site.delete_site()
 296                 # deleted with the site
 297                 #test_site.delete_users()
 298                 continue
 299             else:
 300                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 301                 test_site.create_site()
 302                 test_site.create_users()
 303         return True
 304
 305     def nodes (self, options):
 306         return self.do_nodes(options)
 307     def clean_nodes (self, options):
 308         return self.do_nodes(options,action="delete")
 309
 310     def do_nodes (self, options,action="add"):
 311         for site_spec in self.plc_spec['sites']:
 312             test_site = TestSite (self,site_spec)
 313             if action != "add":
 314                 utils.header("Deleting nodes in site %s"%test_site.name())
 315                 for node_spec in site_spec['nodes']:
 316                     test_node=TestNode(self,test_site,node_spec)
 317                     utils.header("Deleting %s"%test_node.name())
 318                     test_node.delete_node()
 319             else:
 320                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 321                 for node_spec in site_spec['nodes']:
 322                     utils.show_spec('Creating node %s'%node_spec,node_spec)
 323                     test_node = TestNode (self,test_site,node_spec)
 324                     test_node.create_node ()
 325         return True
 326
 327     # create nodegroups if needed, and populate
 328     # no need for a clean_nodegroups if we are careful enough
 329     def nodegroups (self, options):
 330         # 1st pass to scan contents
 331         groups_dict = {}
 332         for site_spec in self.plc_spec['sites']:
 333             test_site = TestSite (self,site_spec)
 334             for node_spec in site_spec['nodes']:
 335                 test_node=TestNode (self,test_site,node_spec)
 336                 if node_spec.has_key('nodegroups'):
 337                     nodegroupnames=node_spec['nodegroups']
 338                     if isinstance(nodegroupnames,StringTypes):
 339                         nodegroupnames = [ nodegroupnames ]
 340                     for nodegroupname in nodegroupnames:
 341                         if not groups_dict.has_key(nodegroupname):
 342                             groups_dict[nodegroupname]=[]
 343                         groups_dict[nodegroupname].append(test_node.name())
 344         auth=self.auth_root()
 345         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 346             try:
 347                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 348             except:
 349                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 350             for node in group_nodes:
 351                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 352         return True
 353
 354     def all_hostnames (self) :
 355         hostnames = []
 356         for site_spec in self.plc_spec['sites']:
 357             hostnames += [ node_spec['node_fields']['hostname'] \
 358                            for node_spec in site_spec['nodes'] ]
 359         return hostnames
 360
 361     # gracetime : during the first <gracetime> minutes nothing gets printed
 362     def do_check_nodesStatus (self, minutes, gracetime=2):
 363         # compute timeout
 364         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 365         graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
 366         # the nodes that haven't checked yet - start with a full list and shrink over time
 367         tocheck = self.all_hostnames()
 368         utils.header("checking nodes %r"%tocheck)
 369         # create a dict hostname -> status
 370         status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
 371         while tocheck:
 372             # get their status
 373             tocheck_status=self.server.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
 374             # update status
 375             for array in tocheck_status:
 376                 hostname=array['hostname']
 377                 boot_state=array['boot_state']
 378                 if boot_state == 'boot':
 379                     utils.header ("%s has reached the 'boot' state"%hostname)
 380                 else:
 381                     # if it's a real node, never mind
 382                     (site_spec,node_spec)=self.locate_node(hostname)
 383                     test_node = TestNode(self,site_spec,node_spec)
 384                     if test_node.is_real():
 385                         utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
 386                         # let's cheat
 387                         boot_state = 'boot'
 388                     if datetime.datetime.now() > graceout:
 389                         utils.header ("%s still in '%s' state"%(hostname,boot_state))
 390                         graceout=datetime.datetime.now()+datetime.timedelta(1)
 391                 status[hostname] = boot_state
 392             # refresh tocheck
 393             tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
 394             if not tocheck:
 395                 return True
 396             if datetime.datetime.now() > timeout:
 397                 for hostname in tocheck:
 398                     utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
 399                 return False
 400             # otherwise, sleep for a while
 401             time.sleep(15)
 402         # only useful in empty plcs
 403         return True
 404
 405     def check_nodesStatus(self,options):
 406         return self.do_check_nodesStatus(minutes=5)
 407
 408     #to scan and store the nodes's public keys and avoid to ask for confirmation when  ssh
 409     def scan_publicKeys(self,hostnames):
 410         try:
 411             temp_knownhosts="/root/known_hosts"
 412             remote_knownhosts="/root/.ssh/known_hosts"
 413             self.run_in_host("touch %s"%temp_knownhosts )
 414             for hostname in hostnames:
 415                 utils.header("Scan Public %s key and store it in the known_host file(under the root image) "%hostname)
 416                 scan=self.run_in_host('ssh-keyscan -t rsa %s >> %s '%(hostname,temp_knownhosts))
 417             #Store the public keys in the right root image
 418             self.copy_in_guest(temp_knownhosts,remote_knownhosts,True)
 419             #clean the temp keys file used
 420             self.run_in_host('rm -f  %s '%temp_knownhosts )
 421         except Exception, err:
 422             print err
 423
 424     def do_check_nodesSsh(self,minutes):
 425         # compute timeout
 426         timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
 427         #graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
 428         tocheck = self.all_hostnames()
 429         self.scan_publicKeys(tocheck)
 430         utils.header("checking Connectivity on nodes %r"%tocheck)
 431         while tocheck:
 432             for hostname in tocheck:
 433                 # try to ssh in nodes
 434                 access=self.run_in_guest('ssh -i /etc/planetlab/root_ssh_key.rsa root@%s date'%hostname )
 435                 if (not access):
 436                     utils.header('The node %s is sshable -->'%hostname)
 437                     # refresh tocheck
 438                     tocheck.remove(hostname)
 439             if not tocheck:
 440                 return True
 441             if datetime.datetime.now() > timeout:
 442                 for hostname in tocheck:
 443                     utils.header("FAILURE to ssh into %s"%hostname)
 444                 return False
 445             # otherwise, sleep for a while
 446             time.sleep(15)
 447         # only useful in empty plcs
 448         return True
 449
 450     def check_nodesConnectivity(self, options):
 451         return  self.do_check_nodesSsh(minutes=2)
 452
 453     def standby(self,options):
 454         #Method for waiting a while when nodes are booting and being sshable,giving time to NM to be up
 455         utils.header('Entering in StanbdBy mode at %s'%datetime.datetime.now())
 456         time.sleep(900)
 457         utils.header('Exist StandBy mode at %s'%datetime.datetime.now())
 458         return True
 459
 460     def bootcd (self, options):
 461         for site_spec in self.plc_spec['sites']:
 462             test_site = TestSite (self,site_spec)
 463             for node_spec in site_spec['nodes']:
 464                 test_node=TestNode (self,test_site,node_spec)
 465                 test_node.create_boot_cd(options.path)
 466         return True
 467
 468     def initscripts (self, options):
 469         for initscript in self.plc_spec['initscripts']:
 470             utils.show_spec('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 471             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 472         return True
 473
 474     def slices (self, options):
 475         return self.do_slices()
 476
 477     def clean_slices (self, options):
 478         return self.do_slices("delete")
 479
 480     def do_slices (self,  action="add"):
 481         for slice in self.plc_spec['slices']:
 482             site_spec = self.locate_site (slice['sitename'])
 483             test_site = TestSite(self,site_spec)
 484             test_slice=TestSlice(self,test_site,slice)
 485             if action != "add":
 486                 utils.header("Deleting slices in site %s"%test_site.name())
 487                 test_slice.delete_slice()
 488             else:
 489                 utils.show_spec("Creating slice",slice)
 490                 test_slice.create_slice()
 491                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
 492         return True
 493
 494     def check_slices(self, options):
 495         for slice_spec in self.plc_spec['slices']:
 496             site_spec = self.locate_site (slice_spec['sitename'])
 497             test_site = TestSite(self,site_spec)
 498             test_slice=TestSlice(self,test_site,slice_spec)
 499             status=test_slice.do_check_slices(options)
 500             return status
 501
 502     def start_nodes (self, options):
 503         self.kill_all_vmwares()
 504         self.kill_all_qemus()
 505         utils.header("Starting vmware nodes")
 506         for site_spec in self.plc_spec['sites']:
 507             TestSite(self,site_spec).start_nodes (options)
 508         return True
 509
 510     def stop_nodes (self, options):
 511         self.kill_all_vmwares ()
 512         self.kill_all_qemus()
 513         return True
 514
 515     # returns the filename to use for sql dump/restore, using options.dbname if set
 516     def dbfile (self, database, options):
 517         # uses options.dbname if it is found
 518         try:
 519             name=options.dbname
 520             if not isinstance(name,StringTypes):
 521                 raise Exception
 522         except:
 523             t=datetime.datetime.now()
 524             d=t.date()
 525             name=str(d)
 526         return "/root/%s-%s.sql"%(database,name)
 527
 528     def db_dump(self, options):
 529
 530         dump=self.dbfile("planetab4",options)
 531         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 532         utils.header('Dumped planetlab4 database in %s'%dump)
 533         return True
 534
 535     def db_restore(self, options):
 536         dump=self.dbfile("planetab4",options)
 537         ##stop httpd service
 538         self.run_in_guest('service httpd stop')
 539         # xxx - need another wrapper
 540         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 541         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 542         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 543         ##starting httpd service
 544         self.run_in_guest('service httpd start')
 545
 546         utils.header('Database restored from ' + dump)