system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16
  17 # step methods must take (self, options) and return a boolean
  18
  19 class TestPlc:
  20
  21     def __init__ (self,plc_spec):
  22         self.plc_spec=plc_spec
  23         self.path=os.path.dirname(sys.argv[0])
  24         try:
  25             self.vserverip=plc_spec['vserverip']
  26             self.vservername=plc_spec['vservername']
  27             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  28             self.vserver=True
  29         except:
  30             self.vserver=False
  31             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  32         utils.header('Using API url %s'%self.url)
  33         self.server=xmlrpclib.Server(self.url,allow_none=True)
  34
  35     def name(self):
  36         name=self.plc_spec['name']
  37         if self.vserver:
  38             return name+"[%s]"%self.vservername
  39         else:
  40             return name+"[chroot]"
  41
  42     def is_local (self):
  43         return self.plc_spec['hostname'] == 'localhost'
  44
  45     # define the API methods on this object through xmlrpc
  46     # would help, but not strictly necessary
  47     def connect (self):
  48         pass
  49
  50     # build the full command so command gets run in the chroot/vserver
  51     def run_command(self,command):
  52         if self.vserver:
  53             return "vserver %s exec %s"%(self.vservername,command)
  54         else:
  55             return "chroot /plc/root %s"%command
  56
  57     def ssh_command(self,command):
  58         if self.is_local():
  59             return command
  60         else:
  61             return "ssh %s sh -c '\"%s\"'"%(self.plc_spec['hostname'],command)
  62
  63     def full_command(self,command):
  64         return self.ssh_command(self.run_command(command))
  65
  66     def run_in_guest (self,command):
  67         return utils.system(self.full_command(command))
  68     def run_in_host (self,command):
  69         return utils.system(self.ssh_command(command))
  70
  71     # xxx quick n dirty
  72     def run_in_guest_piped (self,local,remote):
  73         return utils.system(local+" | "+self.full_command(remote))
  74
  75     def auth_root (self):
  76         return {'Username':self.plc_spec['PLC_ROOT_USER'],
  77                 'AuthMethod':'password',
  78                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
  79                 'Role' : self.plc_spec['role']
  80                 }
  81     def locate_site (self,sitename):
  82         for site in self.plc_spec['sites']:
  83             if site['site_fields']['name'] == sitename:
  84                 return site
  85             if site['site_fields']['login_base'] == sitename:
  86                 return site
  87         raise Exception,"Cannot locate site %s"%sitename
  88
  89     def locate_key (self,keyname):
  90         for key in self.plc_spec['keys']:
  91             if key['name'] == keyname:
  92                 return key
  93         raise Exception,"Cannot locate key %s"%keyname
  94
  95     def kill_all_vmwares(self):
  96         utils.header('Killing any running vmware or vmplayer instance')
  97         utils.system('pgrep vmware | xargs -r kill')
  98         utils.system('pgrep vmplayer | xargs -r kill ')
  99         utils.system('pgrep vmware | xargs -r kill -9')
 100         utils.system('pgrep vmplayer | xargs -r kill -9')
 101
 102     def kill_all_qemus(self):
 103         for site_spec in self.plc_spec['sites']:
 104             test_site = TestSite (self,site_spec)
 105             for node_spec in site_spec['nodes']:
 106                 test_node=TestNode (self,test_site,node_spec)
 107                 model=node_spec['node_fields']['model']
 108                 host_machine=node_spec['node_fields']['host_machine']
 109                 hostname=node_spec['node_fields']['hostname']
 110                 print model
 111                 if model.find("qemu") >= 0:
 112                     utils.system('ssh root@%s  killall qemu'%host_machine)
 113                     test_node.stop_qemu(host_machine,hostname)
 114
 115     #################### step methods
 116
 117     ### uninstall
 118     def uninstall_chroot(self,options):
 119         self.run_in_host('service plc safestop')
 120         #####detecting the last myplc version installed and remove it
 121         self.run_in_host('rpm -e myplc')
 122         ##### Clean up the /plc directory
 123         self.run_in_host('rm -rf  /plc/data')
 124         return True
 125
 126     def uninstall_vserver(self,options):
 127         self.run_in_host("vserver --silent %s delete"%self.vservername)
 128         return True
 129
 130     def uninstall(self,options):
 131         if self.vserver:
 132             return self.uninstall_vserver(options)
 133         else:
 134             return self.uninstall_chroot(options)
 135
 136     ### install
 137     def install_chroot(self,options):
 138         # nothing to do
 139         return True
 140
 141     # xxx this would not work with hostname != localhost as mylc-init-vserver was extracted locally
 142     def install_vserver(self,options):
 143         # we need build dir for vtest-init-vserver
 144         if self.is_local():
 145             # a full path for the local calls
 146             build_dir=self.path+"/build"
 147         else:
 148             # use a standard name - will be relative to HOME
 149             build_dir="tests-system-build"
 150         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 151         if self.run_in_host(build_checkout) != 0:
 152             raise Exception,"Cannot checkout build dir"
 153         # the repo url is taken from myplc-url
 154         # with the last two steps (i386/myplc...) removed
 155         repo_url = options.myplc_url
 156         repo_url = os.path.dirname(repo_url)
 157         repo_url = os.path.dirname(repo_url)
 158         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 159             (build_dir,self.vservername,repo_url,self.vserverip)
 160         if self.run_in_host(create_vserver) != 0:
 161             raise Exception,"Could not create vserver for %s"%self.vservername
 162         return True
 163
 164     def install(self,options):
 165         if self.vserver:
 166             return self.install_vserver(options)
 167         else:
 168             return self.install_chroot(options)
 169
 170     ### install_rpm
 171     def install_rpm_chroot(self,options):
 172         utils.header('Installing from %s'%options.myplc_url)
 173         url=options.myplc_url
 174         self.run_in_host('rpm -Uvh '+url)
 175         self.run_in_host('service plc mount')
 176         return True
 177
 178     def install_rpm_vserver(self,options):
 179         self.run_in_guest("yum -y install myplc-native")
 180         return True
 181
 182     def install_rpm(self,options):
 183         if self.vserver:
 184             return self.install_rpm_vserver(options)
 185         else:
 186             return self.install_rpm_chroot(options)
 187
 188     ###
 189     def configure(self,options):
 190         tmpname='%s/%s.plc-config-tty'%(options.path,self.name())
 191         fileconf=open(tmpname,'w')
 192         for var in [ 'PLC_NAME',
 193                      'PLC_ROOT_PASSWORD',
 194                      'PLC_ROOT_USER',
 195                      'PLC_MAIL_ENABLED',
 196                      'PLC_MAIL_SUPPORT_ADDRESS',
 197                      'PLC_DB_HOST',
 198                      'PLC_API_HOST',
 199                      'PLC_WWW_HOST',
 200                      'PLC_BOOT_HOST',
 201                      'PLC_NET_DNS1',
 202                      'PLC_NET_DNS2']:
 203             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 204         fileconf.write('w\n')
 205         fileconf.write('q\n')
 206         fileconf.close()
 207         utils.system('cat %s'%tmpname)
 208         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 209         utils.system('rm %s'%tmpname)
 210         return True
 211
 212     # the chroot install is slightly different to this respect
 213     def start(self, options):
 214         if self.vserver:
 215             self.run_in_guest('service plc start')
 216         else:
 217             self.run_in_host('service plc start')
 218         return True
 219
 220     def stop(self, options):
 221         if self.vserver:
 222             self.run_in_guest('service plc stop')
 223         else:
 224             self.run_in_host('service plc stop')
 225         return True
 226
 227     # could use a TestKey class
 228     def store_keys(self, options):
 229         for key_spec in self.plc_spec['keys']:
 230             TestKey(self,key_spec).store_key()
 231         return True
 232
 233     def clean_keys(self, options):
 234         utils.system("rm -rf %s/keys/"%self.path)
 235
 236     def sites (self,options):
 237         return self.do_sites(options)
 238
 239     def clean_sites (self,options):
 240         return self.do_sites(options,action="delete")
 241
 242     def do_sites (self,options,action="add"):
 243         for site_spec in self.plc_spec['sites']:
 244             test_site = TestSite (self,site_spec)
 245             if (action != "add"):
 246                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 247                 test_site.delete_site()
 248                 # deleted with the site
 249                 #test_site.delete_users()
 250                 continue
 251             else:
 252                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 253                 test_site.create_site()
 254                 test_site.create_users()
 255         return True
 256
 257     def nodes (self, options):
 258         return self.do_nodes(options)
 259     def clean_nodes (self, options):
 260         return self.do_nodes(options,action="delete")
 261
 262     def do_nodes (self, options,action="add"):
 263         for site_spec in self.plc_spec['sites']:
 264             test_site = TestSite (self,site_spec)
 265             if action != "add":
 266                 utils.header("Deleting nodes in site %s"%test_site.name())
 267                 for node_spec in site_spec['nodes']:
 268                     test_node=TestNode(self,test_site,node_spec)
 269                     utils.header("Deleting %s"%test_node.name())
 270                     test_node.delete_node()
 271             else:
 272                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 273                 for node_spec in site_spec['nodes']:
 274                     utils.show_spec('Creating node %s'%node_spec,node_spec)
 275                     test_node = TestNode (self,test_site,node_spec)
 276                     test_node.create_node ()
 277         return True
 278
 279     # create nodegroups if needed, and populate
 280     # no need for a clean_nodegroups if we are careful enough
 281     def nodegroups (self, options):
 282         # 1st pass to scan contents
 283         groups_dict = {}
 284         for site_spec in self.plc_spec['sites']:
 285             test_site = TestSite (self,site_spec)
 286             for node_spec in site_spec['nodes']:
 287                 test_node=TestNode (self,test_site,node_spec)
 288                 if node_spec.has_key('nodegroups'):
 289                     nodegroupnames=node_spec['nodegroups']
 290                     if isinstance(nodegroupnames,StringTypes):
 291                         nodegroupnames = [ nodegroupnames ]
 292                     for nodegroupname in nodegroupnames:
 293                         if not groups_dict.has_key(nodegroupname):
 294                             groups_dict[nodegroupname]=[]
 295                         groups_dict[nodegroupname].append(test_node.name())
 296         auth=self.auth_root()
 297         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 298             try:
 299                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 300             except:
 301                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 302             for node in group_nodes:
 303                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 304         return True
 305
 306     def check_nodes(self,options):
 307         time.sleep(10)#Wait for the qemu to mount. Only  matter of display
 308         status=True
 309         start_time = datetime.datetime.now()
 310         dead_time=datetime.datetime.now()+ datetime.timedelta(minutes=5)
 311         booted_nodes=[]
 312         for site_spec in self.plc_spec['sites']:
 313             test_site = TestSite (self,site_spec)
 314             utils.header("Starting checking for nodes in site %s"%self.name())
 315             notfullybooted_nodes=[ node_spec['node_fields']['hostname'] for node_spec in site_spec['nodes'] ]
 316             nbr_nodes= len(notfullybooted_nodes)
 317             while (status):
 318                 for node_spec in site_spec['nodes']:
 319                     hostname=node_spec['node_fields']['hostname']
 320                     if (hostname in notfullybooted_nodes): #to avoid requesting already booted node
 321                         test_node=TestNode (self,test_site,node_spec)
 322                         host_machine=node_spec['node_fields']['host_machine']
 323                         node_status=test_node.get_node_status(hostname,host_machine)
 324                         if (node_status):
 325                             booted_nodes.append(hostname)
 326                             del notfullybooted_nodes[notfullybooted_nodes.index(hostname)]
 327                 if ( not notfullybooted_nodes): break
 328                 elif ( start_time  <= dead_time ) :
 329                     start_time=datetime.datetime.now()+ datetime.timedelta(minutes=2)
 330                     time.sleep(15)
 331                 else: status=False
 332             for nodeup in booted_nodes : utils.header("Node %s correctly installed and booted"%nodeup)
 333             for nodedown  in notfullybooted_nodes : utils.header("Node %s not fully booted"%nodedown)
 334             return status
 335
 336     def bootcd (self, options):
 337         for site_spec in self.plc_spec['sites']:
 338             test_site = TestSite (self,site_spec)
 339             for node_spec in site_spec['nodes']:
 340                 test_node=TestNode (self,test_site,node_spec)
 341                 test_node.create_boot_cd(options.path)
 342         return True
 343
 344     def initscripts (self, options):
 345         for initscript in self.plc_spec['initscripts']:
 346             utils.show_spec('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 347             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 348         return True
 349
 350     def slices (self, options):
 351         return self.do_slices()
 352
 353     def clean_slices (self, options):
 354         return self.do_slices("delete")
 355
 356     ### would need a TestSlice class
 357     def do_slices (self, add_or_delete="add"):
 358         for slice in self.plc_spec['slices']:
 359             site_spec = self.locate_site (slice['sitename'])
 360             test_site = TestSite(self,site_spec)
 361             owner_spec = test_site.locate_user(slice['owner'])
 362             auth = TestUser(self,test_site,owner_spec).auth()
 363             slice_fields = slice['slice_fields']
 364             slice_name = slice_fields['name']
 365             if (add_or_delete == "delete"):
 366                 self.server.DeleteSlice(auth,slice_fields['name'])
 367                 utils.header("Deleted slice %s"%slice_fields['name'])
 368                 continue
 369             utils.show_spec("Creating slice",slice_fields)
 370             self.server.AddSlice(auth,slice_fields)
 371             utils.header('Created Slice %s'%slice_fields['name'])
 372             for username in slice['usernames']:
 373                 user_spec=test_site.locate_user(username)
 374                 test_user=TestUser(self,test_site,user_spec)
 375                 self.server.AddPersonToSlice(auth, test_user.name(), slice_name)
 376
 377             hostnames=[]
 378             for nodename in slice['nodenames']:
 379                 node_spec=test_site.locate_node(nodename)
 380                 test_node=TestNode(self,test_site,node_spec)
 381                 hostnames += [test_node.name()]
 382             utils.header("Adding %r in %s"%(hostnames,slice_name))
 383             self.server.AddSliceToNodes(auth, slice_name, hostnames)
 384             if slice.has_key('initscriptname'):
 385                 isname=slice['initscriptname']
 386                 utils.header("Adding initscript %s in %s"%(isname,slice_name))
 387                 self.server.AddSliceAttribute(self.auth_root(), slice_name,
 388                                               'initscript',isname)
 389         return True
 390
 391     def start_nodes (self, options):
 392         self.kill_all_vmwares()
 393         self.kill_all_qemus()
 394         utils.header("Starting vmware nodes")
 395         for site_spec in self.plc_spec['sites']:
 396             TestSite(self,site_spec).start_nodes (options)
 397         return True
 398
 399     def stop_nodes (self, options):
 400         self.kill_all_vmwares ()
 401         self.kill_all_qemus()
 402         return True
 403
 404     # returns the filename to use for sql dump/restore, using options.dbname if set
 405     def dbfile (self, database, options):
 406         # uses options.dbname if it is found
 407         try:
 408             name=options.dbname
 409             if not isinstance(name,StringTypes):
 410                 raise Exception
 411         except:
 412             t=datetime.datetime.now()
 413             d=t.date()
 414             name=str(d)
 415         return "/root/%s-%s.sql"%(database,name)
 416
 417     def db_dump(self, options):
 418
 419         dump=self.dbfile("planetab4",options)
 420         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 421         utils.header('Dumped planetlab4 database in %s'%dump)
 422         return True
 423
 424     def db_restore(self, options):
 425         dump=self.dbfile("planetab4",options)
 426         ##stop httpd service
 427         self.run_in_guest('service httpd stop')
 428         # xxx - need another wrapper
 429         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 430         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 431         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 432         ##starting httpd service
 433         self.run_in_guest('service httpd start')
 434
 435         utils.header('Database restored from ' + dump)