system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16 from TestSlice import TestSlice
  17
  18 # step methods must take (self, options) and return a boolean
  19
  20 class TestPlc:
  21
  22     def __init__ (self,plc_spec):
  23         self.plc_spec=plc_spec
  24         self.path=os.path.dirname(sys.argv[0])
  25         try:
  26             self.vserverip=plc_spec['vserverip']
  27             self.vservername=plc_spec['vservername']
  28             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  29             self.vserver=True
  30         except:
  31             self.vserver=False
  32             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  33         utils.header('Using API url %s'%self.url)
  34         self.server=xmlrpclib.Server(self.url,allow_none=True)
  35
  36     def name(self):
  37         name=self.plc_spec['name']
  38         if self.vserver:
  39             return name+"[%s]"%self.vservername
  40         else:
  41             return name+"[chroot]"
  42
  43     def is_local (self):
  44         return self.plc_spec['hostname'] == 'localhost'
  45
  46     # define the API methods on this object through xmlrpc
  47     # would help, but not strictly necessary
  48     def connect (self):
  49         pass
  50
  51     # build the full command so command gets run in the chroot/vserver
  52     def run_command(self,command):
  53         if self.vserver:
  54             return "vserver %s exec %s"%(self.vservername,command)
  55         else:
  56             return "chroot /plc/root %s"%command
  57
  58     def ssh_command(self,command):
  59         if self.is_local():
  60             return command
  61         else:
  62             return "ssh %s sh -c '\"%s\"'"%(self.plc_spec['hostname'],command)
  63
  64     def full_command(self,command):
  65         return self.ssh_command(self.run_command(command))
  66
  67     def run_in_guest (self,command):
  68         return utils.system(self.full_command(command))
  69     def run_in_host (self,command):
  70         return utils.system(self.ssh_command(command))
  71
  72     # xxx quick n dirty
  73     def run_in_guest_piped (self,local,remote):
  74         return utils.system(local+" | "+self.full_command(remote))
  75
  76     def auth_root (self):
  77         return {'Username':self.plc_spec['PLC_ROOT_USER'],
  78                 'AuthMethod':'password',
  79                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
  80                 'Role' : self.plc_spec['role']
  81                 }
  82     def locate_site (self,sitename):
  83         for site in self.plc_spec['sites']:
  84             if site['site_fields']['name'] == sitename:
  85                 return site
  86             if site['site_fields']['login_base'] == sitename:
  87                 return site
  88         raise Exception,"Cannot locate site %s"%sitename
  89
  90     def locate_key (self,keyname):
  91         for key in self.plc_spec['keys']:
  92             if key['name'] == keyname:
  93                 return key
  94         raise Exception,"Cannot locate key %s"%keyname
  95
  96     def kill_all_vmwares(self):
  97         utils.header('Killing any running vmware or vmplayer instance')
  98         utils.system('pgrep vmware | xargs -r kill')
  99         utils.system('pgrep vmplayer | xargs -r kill ')
 100         utils.system('pgrep vmware | xargs -r kill -9')
 101         utils.system('pgrep vmplayer | xargs -r kill -9')
 102
 103     def kill_all_qemus(self):
 104         for site_spec in self.plc_spec['sites']:
 105             test_site = TestSite (self,site_spec)
 106             for node_spec in site_spec['nodes']:
 107                 test_node=TestNode (self,test_site,node_spec)
 108                 model=node_spec['node_fields']['model']
 109                 host_box=node_spec['node_fields']['host_box']
 110                 hostname=node_spec['node_fields']['hostname']
 111                 print model
 112                 if model.find("qemu") >= 0:
 113                     utils.system('ssh root@%s  killall qemu'%host_box)
 114                     test_node.stop_qemu(host_box,hostname)
 115
 116     #################### step methods
 117
 118     ### uninstall
 119     def uninstall_chroot(self,options):
 120         self.run_in_host('service plc safestop')
 121         #####detecting the last myplc version installed and remove it
 122         self.run_in_host('rpm -e myplc')
 123         ##### Clean up the /plc directory
 124         self.run_in_host('rm -rf  /plc/data')
 125         return True
 126
 127     def uninstall_vserver(self,options):
 128         self.run_in_host("vserver --silent %s delete"%self.vservername)
 129         return True
 130
 131     def uninstall(self,options):
 132         # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
 133         # it sounds safer to have the former uninstalled too
 134         # so let's call both methods uncondionnally
 135         self.uninstall_vserver(options)
 136         self.uninstall_chroot(options)
 137         return True
 138
 139     ### install
 140     def install_chroot(self,options):
 141         # nothing to do
 142         return True
 143
 144     # xxx this would not work with hostname != localhost as mylc-init-vserver was extracted locally
 145     def install_vserver(self,options):
 146         # we need build dir for vtest-init-vserver
 147         if self.is_local():
 148             # a full path for the local calls
 149             build_dir=self.path+"/build"
 150         else:
 151             # use a standard name - will be relative to HOME
 152             build_dir="tests-system-build"
 153         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 154         if self.run_in_host(build_checkout) != 0:
 155             raise Exception,"Cannot checkout build dir"
 156         # the repo url is taken from myplc-url
 157         # with the last two steps (i386/myplc...) removed
 158         repo_url = options.myplc_url
 159         repo_url = os.path.dirname(repo_url)
 160         repo_url = os.path.dirname(repo_url)
 161         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 162             (build_dir,self.vservername,repo_url,self.vserverip)
 163         if self.run_in_host(create_vserver) != 0:
 164             raise Exception,"Could not create vserver for %s"%self.vservername
 165         return True
 166
 167     def install(self,options):
 168         if self.vserver:
 169             return self.install_vserver(options)
 170         else:
 171             return self.install_chroot(options)
 172
 173     ### install_rpm
 174     def install_rpm_chroot(self,options):
 175         utils.header('Installing from %s'%options.myplc_url)
 176         url=options.myplc_url
 177         self.run_in_host('rpm -Uvh '+url)
 178         self.run_in_host('service plc mount')
 179         return True
 180
 181     def install_rpm_vserver(self,options):
 182         self.run_in_guest("yum -y install myplc-native")
 183         return True
 184
 185     def install_rpm(self,options):
 186         if self.vserver:
 187             return self.install_rpm_vserver(options)
 188         else:
 189             return self.install_rpm_chroot(options)
 190
 191     ###
 192     def configure(self,options):
 193         tmpname='%s/%s.plc-config-tty'%(options.path,self.name())
 194         fileconf=open(tmpname,'w')
 195         for var in [ 'PLC_NAME',
 196                      'PLC_ROOT_PASSWORD',
 197                      'PLC_ROOT_USER',
 198                      'PLC_MAIL_ENABLED',
 199                      'PLC_MAIL_SUPPORT_ADDRESS',
 200                      'PLC_DB_HOST',
 201                      'PLC_API_HOST',
 202                      'PLC_WWW_HOST',
 203                      'PLC_BOOT_HOST',
 204                      'PLC_NET_DNS1',
 205                      'PLC_NET_DNS2']:
 206             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 207         fileconf.write('w\n')
 208         fileconf.write('q\n')
 209         fileconf.close()
 210         utils.system('cat %s'%tmpname)
 211         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 212         utils.system('rm %s'%tmpname)
 213         return True
 214
 215     # the chroot install is slightly different to this respect
 216     def start(self, options):
 217         if self.vserver:
 218             self.run_in_guest('service plc start')
 219         else:
 220             self.run_in_host('service plc start')
 221         return True
 222
 223     def stop(self, options):
 224         if self.vserver:
 225             self.run_in_guest('service plc stop')
 226         else:
 227             self.run_in_host('service plc stop')
 228         return True
 229
 230     # could use a TestKey class
 231     def store_keys(self, options):
 232         for key_spec in self.plc_spec['keys']:
 233             TestKey(self,key_spec).store_key()
 234         return True
 235
 236     def clean_keys(self, options):
 237         utils.system("rm -rf %s/keys/"%self.path)
 238
 239     def sites (self,options):
 240         return self.do_sites(options)
 241
 242     def clean_sites (self,options):
 243         return self.do_sites(options,action="delete")
 244
 245     def do_sites (self,options,action="add"):
 246         for site_spec in self.plc_spec['sites']:
 247             test_site = TestSite (self,site_spec)
 248             if (action != "add"):
 249                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 250                 test_site.delete_site()
 251                 # deleted with the site
 252                 #test_site.delete_users()
 253                 continue
 254             else:
 255                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 256                 test_site.create_site()
 257                 test_site.create_users()
 258         return True
 259
 260     def nodes (self, options):
 261         return self.do_nodes(options)
 262     def clean_nodes (self, options):
 263         return self.do_nodes(options,action="delete")
 264
 265     def do_nodes (self, options,action="add"):
 266         for site_spec in self.plc_spec['sites']:
 267             test_site = TestSite (self,site_spec)
 268             if action != "add":
 269                 utils.header("Deleting nodes in site %s"%test_site.name())
 270                 for node_spec in site_spec['nodes']:
 271                     test_node=TestNode(self,test_site,node_spec)
 272                     utils.header("Deleting %s"%test_node.name())
 273                     test_node.delete_node()
 274             else:
 275                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 276                 for node_spec in site_spec['nodes']:
 277                     utils.show_spec('Creating node %s'%node_spec,node_spec)
 278                     test_node = TestNode (self,test_site,node_spec)
 279                     test_node.create_node ()
 280         return True
 281
 282     # create nodegroups if needed, and populate
 283     # no need for a clean_nodegroups if we are careful enough
 284     def nodegroups (self, options):
 285         # 1st pass to scan contents
 286         groups_dict = {}
 287         for site_spec in self.plc_spec['sites']:
 288             test_site = TestSite (self,site_spec)
 289             for node_spec in site_spec['nodes']:
 290                 test_node=TestNode (self,test_site,node_spec)
 291                 if node_spec.has_key('nodegroups'):
 292                     nodegroupnames=node_spec['nodegroups']
 293                     if isinstance(nodegroupnames,StringTypes):
 294                         nodegroupnames = [ nodegroupnames ]
 295                     for nodegroupname in nodegroupnames:
 296                         if not groups_dict.has_key(nodegroupname):
 297                             groups_dict[nodegroupname]=[]
 298                         groups_dict[nodegroupname].append(test_node.name())
 299         auth=self.auth_root()
 300         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 301             try:
 302                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 303             except:
 304                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 305             for node in group_nodes:
 306                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 307         return True
 308
 309     def check_nodes(self,options):
 310         time.sleep(10)#Wait for the qemu to mount. Only  matter of display
 311         status=True
 312         start_time = datetime.datetime.now()
 313         dead_time=datetime.datetime.now()+ datetime.timedelta(minutes=5)
 314         booted_nodes=[]
 315         for site_spec in self.plc_spec['sites']:
 316             test_site = TestSite (self,site_spec)
 317             utils.header("Starting checking for nodes in site %s"%self.name())
 318             notfullybooted_nodes=[ node_spec['node_fields']['hostname'] for node_spec in site_spec['nodes'] ]
 319             nbr_nodes= len(notfullybooted_nodes)
 320             while (status):
 321                 for node_spec in site_spec['nodes']:
 322                     hostname=node_spec['node_fields']['hostname']
 323                     if (hostname in notfullybooted_nodes): #to avoid requesting already booted node
 324                         test_node=TestNode (self,test_site,node_spec)
 325                         host_box=node_spec['node_fields']['host_box']
 326                         node_status=test_node.get_node_status(hostname,host_box)
 327                         if (node_status):
 328                             booted_nodes.append(hostname)
 329                             del notfullybooted_nodes[notfullybooted_nodes.index(hostname)]
 330                 if ( not notfullybooted_nodes): break
 331                 elif ( start_time  <= dead_time ) :
 332                     start_time=datetime.datetime.now()+ datetime.timedelta(minutes=2)
 333                     time.sleep(15)
 334                 else: status=False
 335             for nodeup in booted_nodes : utils.header("Node %s correctly installed and booted"%nodeup)
 336             for nodedown  in notfullybooted_nodes : utils.header("Node %s not fully booted"%nodedown)
 337             return status
 338
 339     def bootcd (self, options):
 340         for site_spec in self.plc_spec['sites']:
 341             test_site = TestSite (self,site_spec)
 342             for node_spec in site_spec['nodes']:
 343                 test_node=TestNode (self,test_site,node_spec)
 344                 test_node.create_boot_cd(options.path)
 345         return True
 346
 347     def initscripts (self, options):
 348         for initscript in self.plc_spec['initscripts']:
 349             utils.show_spec('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 350             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 351         return True
 352
 353     def slices (self, options):
 354         return self.do_slices()
 355
 356     def clean_slices (self, options):
 357         return self.do_slices("delete")
 358
 359     def do_slices (self,  action="add"):
 360         for slice in self.plc_spec['slices']:
 361             site_spec = self.locate_site (slice['sitename'])
 362             test_site = TestSite(self,site_spec)
 363             test_slice=TestSlice(self,test_site,slice)
 364             if action != "add":
 365                 utils.header("Deleting slices in site %s"%test_site.name())
 366                 test_slice.delete_slice()
 367             else:
 368                 utils.show_spec("Creating slice",slice)
 369                 test_slice.create_slice()
 370                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
 371         return True
 372
 373     def check_slices(self, options):
 374         for slice_spec in self.plc_spec['slices']:
 375             site_spec = self.locate_site (slice_spec['sitename'])
 376             test_site = TestSite(self,site_spec)
 377             test_slice=TestSlice(self,test_site,slice_spec)
 378             status=test_slice.do_check_slices()
 379             return status
 380
 381     def start_nodes (self, options):
 382         self.kill_all_vmwares()
 383         self.kill_all_qemus()
 384         utils.header("Starting vmware nodes")
 385         for site_spec in self.plc_spec['sites']:
 386             TestSite(self,site_spec).start_nodes (options)
 387         return True
 388
 389     def stop_nodes (self, options):
 390         self.kill_all_vmwares ()
 391         self.kill_all_qemus()
 392         return True
 393
 394     # returns the filename to use for sql dump/restore, using options.dbname if set
 395     def dbfile (self, database, options):
 396         # uses options.dbname if it is found
 397         try:
 398             name=options.dbname
 399             if not isinstance(name,StringTypes):
 400                 raise Exception
 401         except:
 402             t=datetime.datetime.now()
 403             d=t.date()
 404             name=str(d)
 405         return "/root/%s-%s.sql"%(database,name)
 406
 407     def db_dump(self, options):
 408
 409         dump=self.dbfile("planetab4",options)
 410         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 411         utils.header('Dumped planetlab4 database in %s'%dump)
 412         return True
 413
 414     def db_restore(self, options):
 415         dump=self.dbfile("planetab4",options)
 416         ##stop httpd service
 417         self.run_in_guest('service httpd stop')
 418         # xxx - need another wrapper
 419         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 420         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 421         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 422         ##starting httpd service
 423         self.run_in_guest('service httpd start')
 424
 425         utils.header('Database restored from ' + dump)