system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16 from TestSlice import TestSlice
  17
  18 # step methods must take (self, options) and return a boolean
  19
  20 class TestPlc:
  21
  22     def __init__ (self,plc_spec):
  23         self.plc_spec=plc_spec
  24         self.path=os.path.dirname(sys.argv[0])
  25         try:
  26             self.vserverip=plc_spec['vserverip']
  27             self.vservername=plc_spec['vservername']
  28             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  29             self.vserver=True
  30         except:
  31             self.vserver=False
  32             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  33         utils.header('Using API url %s'%self.url)
  34         self.server=xmlrpclib.Server(self.url,allow_none=True)
  35
  36     def name(self):
  37         name=self.plc_spec['name']
  38         if self.vserver:
  39             return name+"[%s]"%self.vservername
  40         else:
  41             return name+"[chroot]"
  42
  43     def is_local (self):
  44         return self.plc_spec['hostname'] == 'localhost'
  45
  46     # define the API methods on this object through xmlrpc
  47     # would help, but not strictly necessary
  48     def connect (self):
  49         pass
  50
  51     # build the full command so command gets run in the chroot/vserver
  52     def run_command(self,command):
  53         if self.vserver:
  54             return "vserver %s exec %s"%(self.vservername,command)
  55         else:
  56             return "chroot /plc/root %s"%command
  57
  58     def ssh_command(self,command):
  59         if self.is_local():
  60             return command
  61         else:
  62             return "ssh %s sh -c '\"%s\"'"%(self.plc_spec['hostname'],command)
  63
  64     def full_command(self,command):
  65         return self.ssh_command(self.run_command(command))
  66
  67     def run_in_guest (self,command):
  68         return utils.system(self.full_command(command))
  69     def run_in_host (self,command):
  70         return utils.system(self.ssh_command(command))
  71
  72     # xxx quick n dirty
  73     def run_in_guest_piped (self,local,remote):
  74         return utils.system(local+" | "+self.full_command(remote))
  75
  76     def auth_root (self):
  77         return {'Username':self.plc_spec['PLC_ROOT_USER'],
  78                 'AuthMethod':'password',
  79                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
  80                 'Role' : self.plc_spec['role']
  81                 }
  82     def locate_site (self,sitename):
  83         for site in self.plc_spec['sites']:
  84             if site['site_fields']['name'] == sitename:
  85                 return site
  86             if site['site_fields']['login_base'] == sitename:
  87                 return site
  88         raise Exception,"Cannot locate site %s"%sitename
  89
  90     def locate_key (self,keyname):
  91         for key in self.plc_spec['keys']:
  92             if key['name'] == keyname:
  93                 return key
  94         raise Exception,"Cannot locate key %s"%keyname
  95
  96     def kill_all_vmwares(self):
  97         utils.header('Killing any running vmware or vmplayer instance')
  98         utils.system('pgrep vmware | xargs -r kill')
  99         utils.system('pgrep vmplayer | xargs -r kill ')
 100         utils.system('pgrep vmware | xargs -r kill -9')
 101         utils.system('pgrep vmplayer | xargs -r kill -9')
 102
 103     def kill_all_qemus(self):
 104         for site_spec in self.plc_spec['sites']:
 105             test_site = TestSite (self,site_spec)
 106             for node_spec in site_spec['nodes']:
 107                 test_node=TestNode (self,test_site,node_spec)
 108                 model=node_spec['node_fields']['model']
 109                 host_box=node_spec['node_fields']['host_box']
 110                 hostname=node_spec['node_fields']['hostname']
 111                 print model
 112                 if model.find("qemu") >= 0:
 113                     utils.system('ssh root@%s  killall qemu'%host_box)
 114                     test_node.stop_qemu(host_box,hostname)
 115
 116     #################### step methods
 117
 118     ### uninstall
 119     def uninstall_chroot(self,options):
 120         self.run_in_host('service plc safestop')
 121         #####detecting the last myplc version installed and remove it
 122         self.run_in_host('rpm -e myplc')
 123         ##### Clean up the /plc directory
 124         self.run_in_host('rm -rf  /plc/data')
 125         ##### stop any running vservers
 126         self.run_in_host('for vserver in $(cd /vservers ; ls) ; do vserver $vserver stop ; done')
 127         return True
 128
 129     def uninstall_vserver(self,options):
 130         self.run_in_host("vserver --silent %s delete"%self.vservername)
 131         return True
 132
 133     def uninstall(self,options):
 134         # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
 135         # it sounds safer to have the former uninstalled too
 136         # now the vserver method cannot be invoked for chroot instances as vservername is required
 137         if self.vserver:
 138             self.uninstall_vserver(options)
 139             self.uninstall_chroot(options)
 140         else:
 141             self.uninstall_chroot(options)
 142         return True
 143
 144     ### install
 145     def install_chroot(self,options):
 146         # nothing to do
 147         return True
 148
 149     # xxx this would not work with hostname != localhost as mylc-init-vserver was extracted locally
 150     def install_vserver(self,options):
 151         # we need build dir for vtest-init-vserver
 152         if self.is_local():
 153             # a full path for the local calls
 154             build_dir=self.path+"/build"
 155         else:
 156             # use a standard name - will be relative to HOME
 157             build_dir="tests-system-build"
 158         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 159         if self.run_in_host(build_checkout) != 0:
 160             raise Exception,"Cannot checkout build dir"
 161         # the repo url is taken from myplc-url
 162         # with the last two steps (i386/myplc...) removed
 163         repo_url = options.myplc_url
 164         repo_url = os.path.dirname(repo_url)
 165         repo_url = os.path.dirname(repo_url)
 166         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 167             (build_dir,self.vservername,repo_url,self.vserverip)
 168         if self.run_in_host(create_vserver) != 0:
 169             raise Exception,"Could not create vserver for %s"%self.vservername
 170         return True
 171
 172     def install(self,options):
 173         if self.vserver:
 174             return self.install_vserver(options)
 175         else:
 176             return self.install_chroot(options)
 177
 178     ### install_rpm
 179     def install_rpm_chroot(self,options):
 180         utils.header('Installing from %s'%options.myplc_url)
 181         url=options.myplc_url
 182         self.run_in_host('rpm -Uvh '+url)
 183         self.run_in_host('service plc mount')
 184         return True
 185
 186     def install_rpm_vserver(self,options):
 187         self.run_in_guest("yum -y install myplc-native")
 188         return True
 189
 190     def install_rpm(self,options):
 191         if self.vserver:
 192             return self.install_rpm_vserver(options)
 193         else:
 194             return self.install_rpm_chroot(options)
 195
 196     ###
 197     def configure(self,options):
 198         tmpname='%s/%s.plc-config-tty'%(options.path,self.name())
 199         fileconf=open(tmpname,'w')
 200         for var in [ 'PLC_NAME',
 201                      'PLC_ROOT_PASSWORD',
 202                      'PLC_ROOT_USER',
 203                      'PLC_MAIL_ENABLED',
 204                      'PLC_MAIL_SUPPORT_ADDRESS',
 205                      'PLC_DB_HOST',
 206                      'PLC_API_HOST',
 207                      'PLC_WWW_HOST',
 208                      'PLC_BOOT_HOST',
 209                      'PLC_NET_DNS1',
 210                      'PLC_NET_DNS2']:
 211             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 212         fileconf.write('w\n')
 213         fileconf.write('q\n')
 214         fileconf.close()
 215         utils.system('cat %s'%tmpname)
 216         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 217         utils.system('rm %s'%tmpname)
 218         return True
 219
 220     # the chroot install is slightly different to this respect
 221     def start(self, options):
 222         if self.vserver:
 223             self.run_in_guest('service plc start')
 224         else:
 225             self.run_in_host('service plc start')
 226         return True
 227
 228     def stop(self, options):
 229         if self.vserver:
 230             self.run_in_guest('service plc stop')
 231         else:
 232             self.run_in_host('service plc stop')
 233         return True
 234
 235     # could use a TestKey class
 236     def store_keys(self, options):
 237         for key_spec in self.plc_spec['keys']:
 238             TestKey(self,key_spec).store_key()
 239         return True
 240
 241     def clean_keys(self, options):
 242         utils.system("rm -rf %s/keys/"%self.path)
 243
 244     def sites (self,options):
 245         return self.do_sites(options)
 246
 247     def clean_sites (self,options):
 248         return self.do_sites(options,action="delete")
 249
 250     def do_sites (self,options,action="add"):
 251         for site_spec in self.plc_spec['sites']:
 252             test_site = TestSite (self,site_spec)
 253             if (action != "add"):
 254                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 255                 test_site.delete_site()
 256                 # deleted with the site
 257                 #test_site.delete_users()
 258                 continue
 259             else:
 260                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 261                 test_site.create_site()
 262                 test_site.create_users()
 263         return True
 264
 265     def nodes (self, options):
 266         return self.do_nodes(options)
 267     def clean_nodes (self, options):
 268         return self.do_nodes(options,action="delete")
 269
 270     def do_nodes (self, options,action="add"):
 271         for site_spec in self.plc_spec['sites']:
 272             test_site = TestSite (self,site_spec)
 273             if action != "add":
 274                 utils.header("Deleting nodes in site %s"%test_site.name())
 275                 for node_spec in site_spec['nodes']:
 276                     test_node=TestNode(self,test_site,node_spec)
 277                     utils.header("Deleting %s"%test_node.name())
 278                     test_node.delete_node()
 279             else:
 280                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 281                 for node_spec in site_spec['nodes']:
 282                     utils.show_spec('Creating node %s'%node_spec,node_spec)
 283                     test_node = TestNode (self,test_site,node_spec)
 284                     test_node.create_node ()
 285         return True
 286
 287     # create nodegroups if needed, and populate
 288     # no need for a clean_nodegroups if we are careful enough
 289     def nodegroups (self, options):
 290         # 1st pass to scan contents
 291         groups_dict = {}
 292         for site_spec in self.plc_spec['sites']:
 293             test_site = TestSite (self,site_spec)
 294             for node_spec in site_spec['nodes']:
 295                 test_node=TestNode (self,test_site,node_spec)
 296                 if node_spec.has_key('nodegroups'):
 297                     nodegroupnames=node_spec['nodegroups']
 298                     if isinstance(nodegroupnames,StringTypes):
 299                         nodegroupnames = [ nodegroupnames ]
 300                     for nodegroupname in nodegroupnames:
 301                         if not groups_dict.has_key(nodegroupname):
 302                             groups_dict[nodegroupname]=[]
 303                         groups_dict[nodegroupname].append(test_node.name())
 304         auth=self.auth_root()
 305         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 306             try:
 307                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 308             except:
 309                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 310             for node in group_nodes:
 311                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 312         return True
 313
 314     def check_nodes(self,options):
 315         time.sleep(10)#Wait for the qemu to mount. Only  matter of display
 316         status=True
 317         start_time = datetime.datetime.now()
 318         dead_time=datetime.datetime.now()+ datetime.timedelta(minutes=5)
 319         booted_nodes=[]
 320         for site_spec in self.plc_spec['sites']:
 321             test_site = TestSite (self,site_spec)
 322             utils.header("Starting checking for nodes in site %s"%self.name())
 323             notfullybooted_nodes=[ node_spec['node_fields']['hostname'] for node_spec in site_spec['nodes'] ]
 324             nbr_nodes= len(notfullybooted_nodes)
 325             while (status):
 326                 for node_spec in site_spec['nodes']:
 327                     hostname=node_spec['node_fields']['hostname']
 328                     if (hostname in notfullybooted_nodes): #to avoid requesting already booted node
 329                         test_node=TestNode (self,test_site,node_spec)
 330                         host_box=node_spec['node_fields']['host_box']
 331                         node_status=test_node.get_node_status(hostname,host_box)
 332                         if (node_status):
 333                             booted_nodes.append(hostname)
 334                             del notfullybooted_nodes[notfullybooted_nodes.index(hostname)]
 335                 if ( not notfullybooted_nodes): break
 336                 elif ( start_time  <= dead_time ) :
 337                     start_time=datetime.datetime.now()+ datetime.timedelta(minutes=2)
 338                     time.sleep(15)
 339                 else: status=False
 340             for nodeup in booted_nodes : utils.header("Node %s correctly installed and booted"%nodeup)
 341             for nodedown  in notfullybooted_nodes : utils.header("Node %s not fully booted"%nodedown)
 342             return status
 343
 344     def bootcd (self, options):
 345         for site_spec in self.plc_spec['sites']:
 346             test_site = TestSite (self,site_spec)
 347             for node_spec in site_spec['nodes']:
 348                 test_node=TestNode (self,test_site,node_spec)
 349                 test_node.create_boot_cd(options.path)
 350         return True
 351
 352     def initscripts (self, options):
 353         for initscript in self.plc_spec['initscripts']:
 354             utils.show_spec('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 355             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 356         return True
 357
 358     def slices (self, options):
 359         return self.do_slices()
 360
 361     def clean_slices (self, options):
 362         return self.do_slices("delete")
 363
 364     def do_slices (self,  action="add"):
 365         for slice in self.plc_spec['slices']:
 366             site_spec = self.locate_site (slice['sitename'])
 367             test_site = TestSite(self,site_spec)
 368             test_slice=TestSlice(self,test_site,slice)
 369             if action != "add":
 370                 utils.header("Deleting slices in site %s"%test_site.name())
 371                 test_slice.delete_slice()
 372             else:
 373                 utils.show_spec("Creating slice",slice)
 374                 test_slice.create_slice()
 375                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
 376         return True
 377
 378     def check_slices(self, options):
 379         for slice_spec in self.plc_spec['slices']:
 380             site_spec = self.locate_site (slice_spec['sitename'])
 381             test_site = TestSite(self,site_spec)
 382             test_slice=TestSlice(self,test_site,slice_spec)
 383             status=test_slice.do_check_slices()
 384             return status
 385
 386     def start_nodes (self, options):
 387         self.kill_all_vmwares()
 388         self.kill_all_qemus()
 389         utils.header("Starting vmware nodes")
 390         for site_spec in self.plc_spec['sites']:
 391             TestSite(self,site_spec).start_nodes (options)
 392         return True
 393
 394     def stop_nodes (self, options):
 395         self.kill_all_vmwares ()
 396         self.kill_all_qemus()
 397         return True
 398
 399     # returns the filename to use for sql dump/restore, using options.dbname if set
 400     def dbfile (self, database, options):
 401         # uses options.dbname if it is found
 402         try:
 403             name=options.dbname
 404             if not isinstance(name,StringTypes):
 405                 raise Exception
 406         except:
 407             t=datetime.datetime.now()
 408             d=t.date()
 409             name=str(d)
 410         return "/root/%s-%s.sql"%(database,name)
 411
 412     def db_dump(self, options):
 413
 414         dump=self.dbfile("planetab4",options)
 415         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 416         utils.header('Dumped planetlab4 database in %s'%dump)
 417         return True
 418
 419     def db_restore(self, options):
 420         dump=self.dbfile("planetab4",options)
 421         ##stop httpd service
 422         self.run_in_guest('service httpd stop')
 423         # xxx - need another wrapper
 424         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 425         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 426         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 427         ##starting httpd service
 428         self.run_in_guest('service httpd start')
 429
 430         utils.header('Database restored from ' + dump)