system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16 from TestSlice import TestSlice
  17
  18 # step methods must take (self, options) and return a boolean
  19
  20 class TestPlc:
  21
  22     def __init__ (self,plc_spec):
  23         self.plc_spec=plc_spec
  24         self.path=os.path.dirname(sys.argv[0])
  25         try:
  26             self.vserverip=plc_spec['vserverip']
  27             self.vservername=plc_spec['vservername']
  28             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  29             self.vserver=True
  30         except:
  31             self.vserver=False
  32             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  33         utils.header('Using API url %s'%self.url)
  34         self.server=xmlrpclib.Server(self.url,allow_none=True)
  35
  36     def name(self):
  37         name=self.plc_spec['name']
  38         if self.vserver:
  39             return name+"[%s]"%self.vservername
  40         else:
  41             return name+"[chroot]"
  42
  43     def is_local (self):
  44         return self.plc_spec['hostname'] == 'localhost'
  45
  46     # define the API methods on this object through xmlrpc
  47     # would help, but not strictly necessary
  48     def connect (self):
  49         pass
  50
  51     # build the full command so command gets run in the chroot/vserver
  52     def run_command(self,command):
  53         if self.vserver:
  54             return "vserver %s exec %s"%(self.vservername,command)
  55         else:
  56             return "chroot /plc/root %s"%command
  57
  58     def ssh_command(self,command):
  59         if self.is_local():
  60             return command
  61         else:
  62             return "ssh %s sh -c '\"%s\"'"%(self.plc_spec['hostname'],command)
  63
  64     def full_command(self,command):
  65         return self.ssh_command(self.run_command(command))
  66
  67     def run_in_guest (self,command):
  68         return utils.system(self.full_command(command))
  69     def run_in_host (self,command):
  70         return utils.system(self.ssh_command(command))
  71
  72     # xxx quick n dirty
  73     def run_in_guest_piped (self,local,remote):
  74         return utils.system(local+" | "+self.full_command(remote))
  75
  76     def auth_root (self):
  77         return {'Username':self.plc_spec['PLC_ROOT_USER'],
  78                 'AuthMethod':'password',
  79                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
  80                 'Role' : self.plc_spec['role']
  81                 }
  82     def locate_site (self,sitename):
  83         for site in self.plc_spec['sites']:
  84             if site['site_fields']['name'] == sitename:
  85                 return site
  86             if site['site_fields']['login_base'] == sitename:
  87                 return site
  88         raise Exception,"Cannot locate site %s"%sitename
  89
  90     def locate_key (self,keyname):
  91         for key in self.plc_spec['keys']:
  92             if key['name'] == keyname:
  93                 return key
  94         raise Exception,"Cannot locate key %s"%keyname
  95
  96     def kill_all_vmwares(self):
  97         utils.header('Killing any running vmware or vmplayer instance')
  98         utils.system('pgrep vmware | xargs -r kill')
  99         utils.system('pgrep vmplayer | xargs -r kill ')
 100         utils.system('pgrep vmware | xargs -r kill -9')
 101         utils.system('pgrep vmplayer | xargs -r kill -9')
 102
 103     def kill_all_qemus(self):
 104         for site_spec in self.plc_spec['sites']:
 105             test_site = TestSite (self,site_spec)
 106             for node_spec in site_spec['nodes']:
 107                 test_node=TestNode (self,test_site,node_spec)
 108                 model=node_spec['node_fields']['model']
 109                 host_box=node_spec['node_fields']['host_box']
 110                 hostname=node_spec['node_fields']['hostname']
 111                 print model
 112                 if model.find("qemu") >= 0:
 113                     utils.system('ssh root@%s  killall qemu'%host_box)
 114                     test_node.stop_qemu(host_box,hostname)
 115
 116     #################### step methods
 117
 118     ### uninstall
 119     def uninstall_chroot(self,options):
 120         self.run_in_host('service plc safestop')
 121         #####detecting the last myplc version installed and remove it
 122         self.run_in_host('rpm -e myplc')
 123         ##### Clean up the /plc directory
 124         self.run_in_host('rm -rf  /plc/data')
 125         return True
 126
 127     def uninstall_vserver(self,options):
 128         self.run_in_host("vserver --silent %s delete"%self.vservername)
 129         return True
 130
 131     def uninstall(self,options):
 132         if self.vserver:
 133             return self.uninstall_vserver(options)
 134         else:
 135             return self.uninstall_chroot(options)
 136
 137     ### install
 138     def install_chroot(self,options):
 139         # nothing to do
 140         return True
 141
 142     # xxx this would not work with hostname != localhost as mylc-init-vserver was extracted locally
 143     def install_vserver(self,options):
 144         # we need build dir for vtest-init-vserver
 145         if self.is_local():
 146             # a full path for the local calls
 147             build_dir=self.path+"/build"
 148         else:
 149             # use a standard name - will be relative to HOME
 150             build_dir="tests-system-build"
 151         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 152         if self.run_in_host(build_checkout) != 0:
 153             raise Exception,"Cannot checkout build dir"
 154         # the repo url is taken from myplc-url
 155         # with the last two steps (i386/myplc...) removed
 156         repo_url = options.myplc_url
 157         repo_url = os.path.dirname(repo_url)
 158         repo_url = os.path.dirname(repo_url)
 159         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 160             (build_dir,self.vservername,repo_url,self.vserverip)
 161         if self.run_in_host(create_vserver) != 0:
 162             raise Exception,"Could not create vserver for %s"%self.vservername
 163         return True
 164
 165     def install(self,options):
 166         if self.vserver:
 167             return self.install_vserver(options)
 168         else:
 169             return self.install_chroot(options)
 170
 171     ### install_rpm
 172     def install_rpm_chroot(self,options):
 173         utils.header('Installing from %s'%options.myplc_url)
 174         url=options.myplc_url
 175         self.run_in_host('rpm -Uvh '+url)
 176         self.run_in_host('service plc mount')
 177         return True
 178
 179     def install_rpm_vserver(self,options):
 180         self.run_in_guest("yum -y install myplc-native")
 181         return True
 182
 183     def install_rpm(self,options):
 184         if self.vserver:
 185             return self.install_rpm_vserver(options)
 186         else:
 187             return self.install_rpm_chroot(options)
 188
 189     ###
 190     def configure(self,options):
 191         tmpname='%s/%s.plc-config-tty'%(options.path,self.name())
 192         fileconf=open(tmpname,'w')
 193         for var in [ 'PLC_NAME',
 194                      'PLC_ROOT_PASSWORD',
 195                      'PLC_ROOT_USER',
 196                      'PLC_MAIL_ENABLED',
 197                      'PLC_MAIL_SUPPORT_ADDRESS',
 198                      'PLC_DB_HOST',
 199                      'PLC_API_HOST',
 200                      'PLC_WWW_HOST',
 201                      'PLC_BOOT_HOST',
 202                      'PLC_NET_DNS1',
 203                      'PLC_NET_DNS2']:
 204             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 205         fileconf.write('w\n')
 206         fileconf.write('q\n')
 207         fileconf.close()
 208         utils.system('cat %s'%tmpname)
 209         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 210         utils.system('rm %s'%tmpname)
 211         return True
 212
 213     # the chroot install is slightly different to this respect
 214     def start(self, options):
 215         if self.vserver:
 216             self.run_in_guest('service plc start')
 217         else:
 218             self.run_in_host('service plc start')
 219         return True
 220
 221     def stop(self, options):
 222         if self.vserver:
 223             self.run_in_guest('service plc stop')
 224         else:
 225             self.run_in_host('service plc stop')
 226         return True
 227
 228     # could use a TestKey class
 229     def store_keys(self, options):
 230         for key_spec in self.plc_spec['keys']:
 231             TestKey(self,key_spec).store_key()
 232         return True
 233
 234     def clean_keys(self, options):
 235         utils.system("rm -rf %s/keys/"%self.path)
 236
 237     def sites (self,options):
 238         return self.do_sites(options)
 239
 240     def clean_sites (self,options):
 241         return self.do_sites(options,action="delete")
 242
 243     def do_sites (self,options,action="add"):
 244         for site_spec in self.plc_spec['sites']:
 245             test_site = TestSite (self,site_spec)
 246             if (action != "add"):
 247                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 248                 test_site.delete_site()
 249                 # deleted with the site
 250                 #test_site.delete_users()
 251                 continue
 252             else:
 253                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 254                 test_site.create_site()
 255                 test_site.create_users()
 256         return True
 257
 258     def nodes (self, options):
 259         return self.do_nodes(options)
 260     def clean_nodes (self, options):
 261         return self.do_nodes(options,action="delete")
 262
 263     def do_nodes (self, options,action="add"):
 264         for site_spec in self.plc_spec['sites']:
 265             test_site = TestSite (self,site_spec)
 266             if action != "add":
 267                 utils.header("Deleting nodes in site %s"%test_site.name())
 268                 for node_spec in site_spec['nodes']:
 269                     test_node=TestNode(self,test_site,node_spec)
 270                     utils.header("Deleting %s"%test_node.name())
 271                     test_node.delete_node()
 272             else:
 273                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 274                 for node_spec in site_spec['nodes']:
 275                     utils.show_spec('Creating node %s'%node_spec,node_spec)
 276                     test_node = TestNode (self,test_site,node_spec)
 277                     test_node.create_node ()
 278         return True
 279
 280     # create nodegroups if needed, and populate
 281     # no need for a clean_nodegroups if we are careful enough
 282     def nodegroups (self, options):
 283         # 1st pass to scan contents
 284         groups_dict = {}
 285         for site_spec in self.plc_spec['sites']:
 286             test_site = TestSite (self,site_spec)
 287             for node_spec in site_spec['nodes']:
 288                 test_node=TestNode (self,test_site,node_spec)
 289                 if node_spec.has_key('nodegroups'):
 290                     nodegroupnames=node_spec['nodegroups']
 291                     if isinstance(nodegroupnames,StringTypes):
 292                         nodegroupnames = [ nodegroupnames ]
 293                     for nodegroupname in nodegroupnames:
 294                         if not groups_dict.has_key(nodegroupname):
 295                             groups_dict[nodegroupname]=[]
 296                         groups_dict[nodegroupname].append(test_node.name())
 297         auth=self.auth_root()
 298         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 299             try:
 300                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 301             except:
 302                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 303             for node in group_nodes:
 304                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 305         return True
 306
 307     def check_nodes(self,options):
 308         time.sleep(10)#Wait for the qemu to mount. Only  matter of display
 309         status=True
 310         start_time = datetime.datetime.now()
 311         dead_time=datetime.datetime.now()+ datetime.timedelta(minutes=5)
 312         booted_nodes=[]
 313         for site_spec in self.plc_spec['sites']:
 314             test_site = TestSite (self,site_spec)
 315             utils.header("Starting checking for nodes in site %s"%self.name())
 316             notfullybooted_nodes=[ node_spec['node_fields']['hostname'] for node_spec in site_spec['nodes'] ]
 317             nbr_nodes= len(notfullybooted_nodes)
 318             while (status):
 319                 for node_spec in site_spec['nodes']:
 320                     hostname=node_spec['node_fields']['hostname']
 321                     if (hostname in notfullybooted_nodes): #to avoid requesting already booted node
 322                         test_node=TestNode (self,test_site,node_spec)
 323                         host_box=node_spec['node_fields']['host_box']
 324                         node_status=test_node.get_node_status(hostname,host_box)
 325                         if (node_status):
 326                             booted_nodes.append(hostname)
 327                             del notfullybooted_nodes[notfullybooted_nodes.index(hostname)]
 328                 if ( not notfullybooted_nodes): break
 329                 elif ( start_time  <= dead_time ) :
 330                     start_time=datetime.datetime.now()+ datetime.timedelta(minutes=2)
 331                     time.sleep(15)
 332                 else: status=False
 333             for nodeup in booted_nodes : utils.header("Node %s correctly installed and booted"%nodeup)
 334             for nodedown  in notfullybooted_nodes : utils.header("Node %s not fully booted"%nodedown)
 335             return status
 336
 337     def bootcd (self, options):
 338         for site_spec in self.plc_spec['sites']:
 339             test_site = TestSite (self,site_spec)
 340             for node_spec in site_spec['nodes']:
 341                 test_node=TestNode (self,test_site,node_spec)
 342                 test_node.create_boot_cd(options.path)
 343         return True
 344
 345     def initscripts (self, options):
 346         for initscript in self.plc_spec['initscripts']:
 347             utils.show_spec('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 348             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 349         return True
 350
 351     def slices (self, options):
 352         return self.do_slices()
 353
 354     def clean_slices (self, options):
 355         return self.do_slices("delete")
 356
 357     def do_slices (self,  action="add"):
 358         for slice in self.plc_spec['slices']:
 359             site_spec = self.locate_site (slice['sitename'])
 360             test_site = TestSite(self,site_spec)
 361             test_slice=TestSlice(self,test_site,slice)
 362             if action != "add":
 363                 utils.header("Deleting slices in site %s"%test_site.name())
 364                 test_slice.delete_slice()
 365             else:
 366                 utils.show_spec("Creating slice",slice)
 367                 test_slice.create_slice()
 368                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
 369         return True
 370
 371     def check_slices(self, options):
 372         for slice_spec in self.plc_spec['slices']:
 373             site_spec = self.locate_site (slice_spec['sitename'])
 374             test_site = TestSite(self,site_spec)
 375             test_slice=TestSlice(self,test_site,slice_spec)
 376             status=test_slice.do_check_slices()
 377             return status
 378
 379     def start_nodes (self, options):
 380         self.kill_all_vmwares()
 381         self.kill_all_qemus()
 382         utils.header("Starting vmware nodes")
 383         for site_spec in self.plc_spec['sites']:
 384             TestSite(self,site_spec).start_nodes (options)
 385         return True
 386
 387     def stop_nodes (self, options):
 388         self.kill_all_vmwares ()
 389         self.kill_all_qemus()
 390         return True
 391
 392     # returns the filename to use for sql dump/restore, using options.dbname if set
 393     def dbfile (self, database, options):
 394         # uses options.dbname if it is found
 395         try:
 396             name=options.dbname
 397             if not isinstance(name,StringTypes):
 398                 raise Exception
 399         except:
 400             t=datetime.datetime.now()
 401             d=t.date()
 402             name=str(d)
 403         return "/root/%s-%s.sql"%(database,name)
 404
 405     def db_dump(self, options):
 406
 407         dump=self.dbfile("planetab4",options)
 408         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 409         utils.header('Dumped planetlab4 database in %s'%dump)
 410         return True
 411
 412     def db_restore(self, options):
 413         dump=self.dbfile("planetab4",options)
 414         ##stop httpd service
 415         self.run_in_guest('service httpd stop')
 416         # xxx - need another wrapper
 417         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 418         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 419         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 420         ##starting httpd service
 421         self.run_in_guest('service httpd start')
 422
 423         utils.header('Database restored from ' + dump)