system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16 from TestSlice import TestSlice
  17
  18 # step methods must take (self, options) and return a boolean
  19
  20 class TestPlc:
  21
  22     def __init__ (self,plc_spec):
  23         self.plc_spec=plc_spec
  24         self.path=os.path.dirname(sys.argv[0])
  25         try:
  26             self.vserverip=plc_spec['vserverip']
  27             self.vservername=plc_spec['vservername']
  28             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  29             self.vserver=True
  30         except:
  31             self.vserver=False
  32             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  33         utils.header('Using API url %s'%self.url)
  34         self.server=xmlrpclib.Server(self.url,allow_none=True)
  35
  36     def name(self):
  37         name=self.plc_spec['name']
  38         if self.vserver:
  39             return name+"[%s]"%self.vservername
  40         else:
  41             return name+"[chroot]"
  42
  43     def is_local (self):
  44         return self.plc_spec['hostname'] == 'localhost'
  45
  46     # define the API methods on this object through xmlrpc
  47     # would help, but not strictly necessary
  48     def connect (self):
  49         pass
  50
  51     # build the full command so command gets run in the chroot/vserver
  52     def run_command(self,command):
  53         if self.vserver:
  54             return "vserver %s exec %s"%(self.vservername,command)
  55         else:
  56             return "chroot /plc/root %s"%command
  57
  58     def ssh_command(self,command):
  59         if self.is_local():
  60             return command
  61         else:
  62             return "ssh %s sh -c '\"%s\"'"%(self.plc_spec['hostname'],command)
  63
  64     def full_command(self,command):
  65         return self.ssh_command(self.run_command(command))
  66
  67     def run_in_guest (self,command):
  68         return utils.system(self.full_command(command))
  69     def run_in_host (self,command):
  70         return utils.system(self.ssh_command(command))
  71
  72     # xxx quick n dirty
  73     def run_in_guest_piped (self,local,remote):
  74         return utils.system(local+" | "+self.full_command(remote))
  75
  76     def auth_root (self):
  77         return {'Username':self.plc_spec['PLC_ROOT_USER'],
  78                 'AuthMethod':'password',
  79                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
  80                 'Role' : self.plc_spec['role']
  81                 }
  82     def locate_site (self,sitename):
  83         for site in self.plc_spec['sites']:
  84             if site['site_fields']['name'] == sitename:
  85                 return site
  86             if site['site_fields']['login_base'] == sitename:
  87                 return site
  88         raise Exception,"Cannot locate site %s"%sitename
  89
  90     def locate_key (self,keyname):
  91         for key in self.plc_spec['keys']:
  92             if key['name'] == keyname:
  93                 return key
  94         raise Exception,"Cannot locate key %s"%keyname
  95
  96     def kill_all_vmwares(self):
  97         utils.header('Killing any running vmware or vmplayer instance')
  98         utils.system('pgrep vmware | xargs -r kill')
  99         utils.system('pgrep vmplayer | xargs -r kill ')
 100         utils.system('pgrep vmware | xargs -r kill -9')
 101         utils.system('pgrep vmplayer | xargs -r kill -9')
 102
 103     def kill_all_qemus(self):
 104         for site_spec in self.plc_spec['sites']:
 105             test_site = TestSite (self,site_spec)
 106             for node_spec in site_spec['nodes']:
 107                 test_node=TestNode (self,test_site,node_spec)
 108                 model=node_spec['node_fields']['model']
 109                 host_box=node_spec['node_fields']['host_box']
 110                 hostname=node_spec['node_fields']['hostname']
 111                 print model
 112                 if model.find("qemu") >= 0:
 113                     utils.system('ssh root@%s  killall qemu'%host_box)
 114                     test_node.stop_qemu(host_box,hostname)
 115
 116     #################### step methods
 117
 118     ### uninstall
 119     def uninstall_chroot(self,options):
 120         self.run_in_host('service plc safestop')
 121         #####detecting the last myplc version installed and remove it
 122         self.run_in_host('rpm -e myplc')
 123         ##### Clean up the /plc directory
 124         self.run_in_host('rm -rf  /plc/data')
 125         return True
 126
 127     def uninstall_vserver(self,options):
 128         self.run_in_host("vserver --silent %s delete"%self.vservername)
 129         return True
 130
 131     def uninstall(self,options):
 132         # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
 133         # it sounds safer to have the former uninstalled too
 134         # now the vserver method cannot be invoked for chroot instances as vservername is required
 135         if self.vserver:
 136             self.uninstall_vserver(options)
 137             self.uninstall_chroot(options)
 138         else:
 139             self.uninstall_chroot(options)
 140         return True
 141
 142     ### install
 143     def install_chroot(self,options):
 144         # nothing to do
 145         return True
 146
 147     # xxx this would not work with hostname != localhost as mylc-init-vserver was extracted locally
 148     def install_vserver(self,options):
 149         # we need build dir for vtest-init-vserver
 150         if self.is_local():
 151             # a full path for the local calls
 152             build_dir=self.path+"/build"
 153         else:
 154             # use a standard name - will be relative to HOME
 155             build_dir="tests-system-build"
 156         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 157         if self.run_in_host(build_checkout) != 0:
 158             raise Exception,"Cannot checkout build dir"
 159         # the repo url is taken from myplc-url
 160         # with the last two steps (i386/myplc...) removed
 161         repo_url = options.myplc_url
 162         repo_url = os.path.dirname(repo_url)
 163         repo_url = os.path.dirname(repo_url)
 164         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 165             (build_dir,self.vservername,repo_url,self.vserverip)
 166         if self.run_in_host(create_vserver) != 0:
 167             raise Exception,"Could not create vserver for %s"%self.vservername
 168         return True
 169
 170     def install(self,options):
 171         if self.vserver:
 172             return self.install_vserver(options)
 173         else:
 174             return self.install_chroot(options)
 175
 176     ### install_rpm
 177     def install_rpm_chroot(self,options):
 178         utils.header('Installing from %s'%options.myplc_url)
 179         url=options.myplc_url
 180         self.run_in_host('rpm -Uvh '+url)
 181         self.run_in_host('service plc mount')
 182         return True
 183
 184     def install_rpm_vserver(self,options):
 185         self.run_in_guest("yum -y install myplc-native")
 186         return True
 187
 188     def install_rpm(self,options):
 189         if self.vserver:
 190             return self.install_rpm_vserver(options)
 191         else:
 192             return self.install_rpm_chroot(options)
 193
 194     ###
 195     def configure(self,options):
 196         tmpname='%s/%s.plc-config-tty'%(options.path,self.name())
 197         fileconf=open(tmpname,'w')
 198         for var in [ 'PLC_NAME',
 199                      'PLC_ROOT_PASSWORD',
 200                      'PLC_ROOT_USER',
 201                      'PLC_MAIL_ENABLED',
 202                      'PLC_MAIL_SUPPORT_ADDRESS',
 203                      'PLC_DB_HOST',
 204                      'PLC_API_HOST',
 205                      'PLC_WWW_HOST',
 206                      'PLC_BOOT_HOST',
 207                      'PLC_NET_DNS1',
 208                      'PLC_NET_DNS2']:
 209             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 210         fileconf.write('w\n')
 211         fileconf.write('q\n')
 212         fileconf.close()
 213         utils.system('cat %s'%tmpname)
 214         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 215         utils.system('rm %s'%tmpname)
 216         return True
 217
 218     # the chroot install is slightly different to this respect
 219     def start(self, options):
 220         if self.vserver:
 221             self.run_in_guest('service plc start')
 222         else:
 223             self.run_in_host('service plc start')
 224         return True
 225
 226     def stop(self, options):
 227         if self.vserver:
 228             self.run_in_guest('service plc stop')
 229         else:
 230             self.run_in_host('service plc stop')
 231         return True
 232
 233     # could use a TestKey class
 234     def store_keys(self, options):
 235         for key_spec in self.plc_spec['keys']:
 236             TestKey(self,key_spec).store_key()
 237         return True
 238
 239     def clean_keys(self, options):
 240         utils.system("rm -rf %s/keys/"%self.path)
 241
 242     def sites (self,options):
 243         return self.do_sites(options)
 244
 245     def clean_sites (self,options):
 246         return self.do_sites(options,action="delete")
 247
 248     def do_sites (self,options,action="add"):
 249         for site_spec in self.plc_spec['sites']:
 250             test_site = TestSite (self,site_spec)
 251             if (action != "add"):
 252                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 253                 test_site.delete_site()
 254                 # deleted with the site
 255                 #test_site.delete_users()
 256                 continue
 257             else:
 258                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 259                 test_site.create_site()
 260                 test_site.create_users()
 261         return True
 262
 263     def nodes (self, options):
 264         return self.do_nodes(options)
 265     def clean_nodes (self, options):
 266         return self.do_nodes(options,action="delete")
 267
 268     def do_nodes (self, options,action="add"):
 269         for site_spec in self.plc_spec['sites']:
 270             test_site = TestSite (self,site_spec)
 271             if action != "add":
 272                 utils.header("Deleting nodes in site %s"%test_site.name())
 273                 for node_spec in site_spec['nodes']:
 274                     test_node=TestNode(self,test_site,node_spec)
 275                     utils.header("Deleting %s"%test_node.name())
 276                     test_node.delete_node()
 277             else:
 278                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 279                 for node_spec in site_spec['nodes']:
 280                     utils.show_spec('Creating node %s'%node_spec,node_spec)
 281                     test_node = TestNode (self,test_site,node_spec)
 282                     test_node.create_node ()
 283         return True
 284
 285     # create nodegroups if needed, and populate
 286     # no need for a clean_nodegroups if we are careful enough
 287     def nodegroups (self, options):
 288         # 1st pass to scan contents
 289         groups_dict = {}
 290         for site_spec in self.plc_spec['sites']:
 291             test_site = TestSite (self,site_spec)
 292             for node_spec in site_spec['nodes']:
 293                 test_node=TestNode (self,test_site,node_spec)
 294                 if node_spec.has_key('nodegroups'):
 295                     nodegroupnames=node_spec['nodegroups']
 296                     if isinstance(nodegroupnames,StringTypes):
 297                         nodegroupnames = [ nodegroupnames ]
 298                     for nodegroupname in nodegroupnames:
 299                         if not groups_dict.has_key(nodegroupname):
 300                             groups_dict[nodegroupname]=[]
 301                         groups_dict[nodegroupname].append(test_node.name())
 302         auth=self.auth_root()
 303         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 304             try:
 305                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 306             except:
 307                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 308             for node in group_nodes:
 309                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 310         return True
 311
 312     def check_nodes(self,options):
 313         time.sleep(10)#Wait for the qemu to mount. Only  matter of display
 314         status=True
 315         start_time = datetime.datetime.now()
 316         dead_time=datetime.datetime.now()+ datetime.timedelta(minutes=5)
 317         booted_nodes=[]
 318         for site_spec in self.plc_spec['sites']:
 319             test_site = TestSite (self,site_spec)
 320             utils.header("Starting checking for nodes in site %s"%self.name())
 321             notfullybooted_nodes=[ node_spec['node_fields']['hostname'] for node_spec in site_spec['nodes'] ]
 322             nbr_nodes= len(notfullybooted_nodes)
 323             while (status):
 324                 for node_spec in site_spec['nodes']:
 325                     hostname=node_spec['node_fields']['hostname']
 326                     if (hostname in notfullybooted_nodes): #to avoid requesting already booted node
 327                         test_node=TestNode (self,test_site,node_spec)
 328                         host_box=node_spec['node_fields']['host_box']
 329                         node_status=test_node.get_node_status(hostname,host_box)
 330                         if (node_status):
 331                             booted_nodes.append(hostname)
 332                             del notfullybooted_nodes[notfullybooted_nodes.index(hostname)]
 333                 if ( not notfullybooted_nodes): break
 334                 elif ( start_time  <= dead_time ) :
 335                     start_time=datetime.datetime.now()+ datetime.timedelta(minutes=2)
 336                     time.sleep(15)
 337                 else: status=False
 338             for nodeup in booted_nodes : utils.header("Node %s correctly installed and booted"%nodeup)
 339             for nodedown  in notfullybooted_nodes : utils.header("Node %s not fully booted"%nodedown)
 340             return status
 341
 342     def bootcd (self, options):
 343         for site_spec in self.plc_spec['sites']:
 344             test_site = TestSite (self,site_spec)
 345             for node_spec in site_spec['nodes']:
 346                 test_node=TestNode (self,test_site,node_spec)
 347                 test_node.create_boot_cd(options.path)
 348         return True
 349
 350     def initscripts (self, options):
 351         for initscript in self.plc_spec['initscripts']:
 352             utils.show_spec('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 353             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 354         return True
 355
 356     def slices (self, options):
 357         return self.do_slices()
 358
 359     def clean_slices (self, options):
 360         return self.do_slices("delete")
 361
 362     def do_slices (self,  action="add"):
 363         for slice in self.plc_spec['slices']:
 364             site_spec = self.locate_site (slice['sitename'])
 365             test_site = TestSite(self,site_spec)
 366             test_slice=TestSlice(self,test_site,slice)
 367             if action != "add":
 368                 utils.header("Deleting slices in site %s"%test_site.name())
 369                 test_slice.delete_slice()
 370             else:
 371                 utils.show_spec("Creating slice",slice)
 372                 test_slice.create_slice()
 373                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
 374         return True
 375
 376     def check_slices(self, options):
 377         for slice_spec in self.plc_spec['slices']:
 378             site_spec = self.locate_site (slice_spec['sitename'])
 379             test_site = TestSite(self,site_spec)
 380             test_slice=TestSlice(self,test_site,slice_spec)
 381             status=test_slice.do_check_slices()
 382             return status
 383
 384     def start_nodes (self, options):
 385         self.kill_all_vmwares()
 386         self.kill_all_qemus()
 387         utils.header("Starting vmware nodes")
 388         for site_spec in self.plc_spec['sites']:
 389             TestSite(self,site_spec).start_nodes (options)
 390         return True
 391
 392     def stop_nodes (self, options):
 393         self.kill_all_vmwares ()
 394         self.kill_all_qemus()
 395         return True
 396
 397     # returns the filename to use for sql dump/restore, using options.dbname if set
 398     def dbfile (self, database, options):
 399         # uses options.dbname if it is found
 400         try:
 401             name=options.dbname
 402             if not isinstance(name,StringTypes):
 403                 raise Exception
 404         except:
 405             t=datetime.datetime.now()
 406             d=t.date()
 407             name=str(d)
 408         return "/root/%s-%s.sql"%(database,name)
 409
 410     def db_dump(self, options):
 411
 412         dump=self.dbfile("planetab4",options)
 413         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 414         utils.header('Dumped planetlab4 database in %s'%dump)
 415         return True
 416
 417     def db_restore(self, options):
 418         dump=self.dbfile("planetab4",options)
 419         ##stop httpd service
 420         self.run_in_guest('service httpd stop')
 421         # xxx - need another wrapper
 422         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 423         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 424         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 425         ##starting httpd service
 426         self.run_in_guest('service httpd start')
 427
 428         utils.header('Database restored from ' + dump)