system/TestPlc.py

   1 # $Id$
   2 import os, os.path
   3 import datetime
   4 import time
   5 import sys
   6 import xmlrpclib
   7 import datetime
   8 import traceback
   9 from types import StringTypes
  10
  11 import utils
  12 from TestSite import TestSite
  13 from TestNode import TestNode
  14 from TestUser import TestUser
  15 from TestKey import TestKey
  16 from TestSlice import TestSlice
  17
  18 # step methods must take (self, options) and return a boolean
  19
  20 class TestPlc:
  21
  22     def __init__ (self,plc_spec):
  23         self.plc_spec=plc_spec
  24         self.path=os.path.dirname(sys.argv[0])
  25         try:
  26             self.vserverip=plc_spec['vserverip']
  27             self.vservername=plc_spec['vservername']
  28             self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
  29             self.vserver=True
  30         except:
  31             self.vserver=False
  32             self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
  33         utils.header('Using API url %s'%self.url)
  34         self.server=xmlrpclib.Server(self.url,allow_none=True)
  35
  36     def name(self):
  37         name=self.plc_spec['name']
  38         if self.vserver:
  39             return name+"[%s]"%self.vservername
  40         else:
  41             return name+"[chroot]"
  42
  43     def is_local (self):
  44         return self.plc_spec['hostname'] == 'localhost'
  45
  46     # define the API methods on this object through xmlrpc
  47     # would help, but not strictly necessary
  48     def connect (self):
  49         pass
  50
  51     # build the full command so command gets run in the chroot/vserver
  52     def run_command(self,command):
  53         if self.vserver:
  54             return "vserver %s exec %s"%(self.vservername,command)
  55         else:
  56             return "chroot /plc/root sh -c \\\"%s\\\""%command
  57
  58     def ssh_command(self,command):
  59         if self.is_local():
  60             return command
  61         else:
  62             return "ssh %s sh -c '\"%s\"'"%(self.plc_spec['hostname'],command)
  63
  64     def full_command(self,command):
  65         return self.ssh_command(self.run_command(command))
  66
  67     def run_in_guest (self,command):
  68         return utils.system(self.full_command(command))
  69     def run_in_host (self,command):
  70         return utils.system(self.ssh_command(command))
  71
  72     # xxx quick n dirty
  73     def run_in_guest_piped (self,local,remote):
  74         return utils.system(local+" | "+self.full_command(remote))
  75
  76     def auth_root (self):
  77         return {'Username':self.plc_spec['PLC_ROOT_USER'],
  78                 'AuthMethod':'password',
  79                 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
  80                 'Role' : self.plc_spec['role']
  81                 }
  82     def locate_site (self,sitename):
  83         for site in self.plc_spec['sites']:
  84             if site['site_fields']['name'] == sitename:
  85                 return site
  86             if site['site_fields']['login_base'] == sitename:
  87                 return site
  88         raise Exception,"Cannot locate site %s"%sitename
  89
  90     def locate_key (self,keyname):
  91         for key in self.plc_spec['keys']:
  92             if key['name'] == keyname:
  93                 return key
  94         raise Exception,"Cannot locate key %s"%keyname
  95
  96     def kill_all_vmwares(self):
  97         utils.header('Killing any running vmware or vmplayer instance')
  98         utils.system('pgrep vmware | xargs -r kill')
  99         utils.system('pgrep vmplayer | xargs -r kill ')
 100         utils.system('pgrep vmware | xargs -r kill -9')
 101         utils.system('pgrep vmplayer | xargs -r kill -9')
 102
 103     def kill_all_qemus(self):
 104         for site_spec in self.plc_spec['sites']:
 105             test_site = TestSite (self,site_spec)
 106             for node_spec in site_spec['nodes']:
 107                 TestNode (self,test_site,node_spec).stop_qemu()
 108
 109     def clear_ssh_config (self):
 110         # using ssh -o "BatchMode yes" is too tricky due to quoting - let's use the config
 111         utils.header("Setting BatchMode and StrictHostKeyChecking in ssh config")
 112         self.run_in_guest("sed -i -e '/BatchMode/d' /root/.ssh/config 2> /dev/null")
 113         self.run_in_guest_piped("echo BatchMode yes", "cat >> /root/.ssh/config")
 114         self.run_in_guest("sed -i -e '/StrictHostKeyChecking/d' /root/.ssh/config 2> /dev/null")
 115         self.run_in_guest_piped("echo StrictHostKeyChecking no", "cat >> /root/.ssh/config")
 116
 117     #################### step methods
 118
 119     ### uninstall
 120     def uninstall_chroot(self,options):
 121         self.run_in_host('service plc safestop')
 122         #####detecting the last myplc version installed and remove it
 123         self.run_in_host('rpm -e myplc')
 124         ##### Clean up the /plc directory
 125         self.run_in_host('rm -rf  /plc/data')
 126         ##### stop any running vservers
 127         self.run_in_host('for vserver in $(cd /vservers ; ls) ; do vserver $vserver stop ; done')
 128         return True
 129
 130     def uninstall_vserver(self,options):
 131         self.run_in_host("vserver --silent %s delete"%self.vservername)
 132         return True
 133
 134     def uninstall(self,options):
 135         # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
 136         # it sounds safer to have the former uninstalled too
 137         # now the vserver method cannot be invoked for chroot instances as vservername is required
 138         if self.vserver:
 139             self.uninstall_vserver(options)
 140             self.uninstall_chroot(options)
 141         else:
 142             self.uninstall_chroot(options)
 143         return True
 144
 145     ### install
 146     def install_chroot(self,options):
 147         # nothing to do
 148         return True
 149
 150     # xxx this would not work with hostname != localhost as mylc-init-vserver was extracted locally
 151     def install_vserver(self,options):
 152         # we need build dir for vtest-init-vserver
 153         if self.is_local():
 154             # a full path for the local calls
 155             build_dir=self.path+"/build"
 156         else:
 157             # use a standard name - will be relative to HOME
 158             build_dir="tests-system-build"
 159         build_checkout = "svn checkout %s %s"%(options.build_url,build_dir)
 160         if self.run_in_host(build_checkout) != 0:
 161             raise Exception,"Cannot checkout build dir"
 162         # the repo url is taken from myplc-url
 163         # with the last two steps (i386/myplc...) removed
 164         repo_url = options.myplc_url
 165         repo_url = os.path.dirname(repo_url)
 166         repo_url = os.path.dirname(repo_url)
 167         create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
 168             (build_dir,self.vservername,repo_url,self.vserverip)
 169         if self.run_in_host(create_vserver) != 0:
 170             raise Exception,"Could not create vserver for %s"%self.vservername
 171         return True
 172
 173     def install(self,options):
 174         if self.vserver:
 175             return self.install_vserver(options)
 176         else:
 177             return self.install_chroot(options)
 178
 179     ### install_rpm
 180     def install_rpm_chroot(self,options):
 181         utils.header('Installing from %s'%options.myplc_url)
 182         url=options.myplc_url
 183         self.run_in_host('rpm -Uvh '+url)
 184         self.run_in_host('service plc mount')
 185         return True
 186
 187     def install_rpm_vserver(self,options):
 188         self.run_in_guest("yum -y install myplc-native")
 189         return True
 190
 191     def install_rpm(self,options):
 192         if self.vserver:
 193             return self.install_rpm_vserver(options)
 194         else:
 195             return self.install_rpm_chroot(options)
 196
 197     ###
 198     def configure(self,options):
 199         tmpname='%s/%s.plc-config-tty'%(options.path,self.name())
 200         fileconf=open(tmpname,'w')
 201         for var in [ 'PLC_NAME',
 202                      'PLC_ROOT_PASSWORD',
 203                      'PLC_ROOT_USER',
 204                      'PLC_MAIL_ENABLED',
 205                      'PLC_MAIL_SUPPORT_ADDRESS',
 206                      'PLC_DB_HOST',
 207                      'PLC_API_HOST',
 208                      'PLC_WWW_HOST',
 209                      'PLC_BOOT_HOST',
 210                      'PLC_NET_DNS1',
 211                      'PLC_NET_DNS2']:
 212             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
 213         fileconf.write('w\n')
 214         fileconf.write('q\n')
 215         fileconf.close()
 216         utils.system('cat %s'%tmpname)
 217         self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
 218         utils.system('rm %s'%tmpname)
 219         return True
 220
 221     # the chroot install is slightly different to this respect
 222     def start(self, options):
 223         if self.vserver:
 224             self.run_in_guest('service plc start')
 225         else:
 226             self.run_in_host('service plc start')
 227         return True
 228
 229     def stop(self, options):
 230         if self.vserver:
 231             self.run_in_guest('service plc stop')
 232         else:
 233             self.run_in_host('service plc stop')
 234         return True
 235
 236     # could use a TestKey class
 237     def store_keys(self, options):
 238         for key_spec in self.plc_spec['keys']:
 239             TestKey(self,key_spec).store_key()
 240         return True
 241
 242     def clean_keys(self, options):
 243         utils.system("rm -rf %s/keys/"%self.path)
 244
 245     def sites (self,options):
 246         return self.do_sites(options)
 247
 248     def clean_sites (self,options):
 249         return self.do_sites(options,action="delete")
 250
 251     def do_sites (self,options,action="add"):
 252         for site_spec in self.plc_spec['sites']:
 253             test_site = TestSite (self,site_spec)
 254             if (action != "add"):
 255                 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
 256                 test_site.delete_site()
 257                 # deleted with the site
 258                 #test_site.delete_users()
 259                 continue
 260             else:
 261                 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
 262                 test_site.create_site()
 263                 test_site.create_users()
 264         return True
 265
 266     def nodes (self, options):
 267         return self.do_nodes(options)
 268     def clean_nodes (self, options):
 269         return self.do_nodes(options,action="delete")
 270
 271     def do_nodes (self, options,action="add"):
 272         for site_spec in self.plc_spec['sites']:
 273             test_site = TestSite (self,site_spec)
 274             if action != "add":
 275                 utils.header("Deleting nodes in site %s"%test_site.name())
 276                 for node_spec in site_spec['nodes']:
 277                     test_node=TestNode(self,test_site,node_spec)
 278                     utils.header("Deleting %s"%test_node.name())
 279                     test_node.delete_node()
 280             else:
 281                 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
 282                 for node_spec in site_spec['nodes']:
 283                     utils.show_spec('Creating node %s'%node_spec,node_spec)
 284                     test_node = TestNode (self,test_site,node_spec)
 285                     test_node.create_node ()
 286         return True
 287
 288     # create nodegroups if needed, and populate
 289     # no need for a clean_nodegroups if we are careful enough
 290     def nodegroups (self, options):
 291         # 1st pass to scan contents
 292         groups_dict = {}
 293         for site_spec in self.plc_spec['sites']:
 294             test_site = TestSite (self,site_spec)
 295             for node_spec in site_spec['nodes']:
 296                 test_node=TestNode (self,test_site,node_spec)
 297                 if node_spec.has_key('nodegroups'):
 298                     nodegroupnames=node_spec['nodegroups']
 299                     if isinstance(nodegroupnames,StringTypes):
 300                         nodegroupnames = [ nodegroupnames ]
 301                     for nodegroupname in nodegroupnames:
 302                         if not groups_dict.has_key(nodegroupname):
 303                             groups_dict[nodegroupname]=[]
 304                         groups_dict[nodegroupname].append(test_node.name())
 305         auth=self.auth_root()
 306         for (nodegroupname,group_nodes) in groups_dict.iteritems():
 307             try:
 308                 self.server.GetNodeGroups(auth,{'name':nodegroupname})[0]
 309             except:
 310                 self.server.AddNodeGroup(auth,{'name':nodegroupname})
 311             for node in group_nodes:
 312                 self.server.AddNodeToNodeGroup(auth,node,nodegroupname)
 313         return True
 314
 315     def check_nodes(self,options):
 316         time.sleep(10)#Wait for the qemu to mount. Only  matter of display
 317         status=True
 318         start_time = datetime.datetime.now()
 319         dead_time=datetime.datetime.now()+ datetime.timedelta(minutes=5)
 320         booted_nodes=[]
 321         for site_spec in self.plc_spec['sites']:
 322             test_site = TestSite (self,site_spec)
 323             utils.header("Starting checking for nodes in site %s"%self.name())
 324             notfullybooted_nodes=[ node_spec['node_fields']['hostname'] for node_spec in site_spec['nodes'] ]
 325             nbr_nodes= len(notfullybooted_nodes)
 326             while (status):
 327                 for node_spec in site_spec['nodes']:
 328                     hostname=node_spec['node_fields']['hostname']
 329                     if (hostname in notfullybooted_nodes): #to avoid requesting already booted node
 330                         test_node=TestNode (self,test_site,node_spec)
 331                         host_box=test_node.host_box()
 332                         node_status=test_node.get_node_status(hostname)
 333                         if (node_status):
 334                             booted_nodes.append(hostname)
 335                             del notfullybooted_nodes[notfullybooted_nodes.index(hostname)]
 336                 if ( not notfullybooted_nodes): break
 337                 elif ( start_time  <= dead_time ) :
 338                     start_time=datetime.datetime.now()+ datetime.timedelta(minutes=2)
 339                     time.sleep(15)
 340                 else: status=False
 341             for nodeup in booted_nodes : utils.header("Node %s correctly installed and booted"%nodeup)
 342             for nodedown  in notfullybooted_nodes : utils.header("Node %s not fully booted"%nodedown)
 343             return status
 344
 345     def bootcd (self, options):
 346         for site_spec in self.plc_spec['sites']:
 347             test_site = TestSite (self,site_spec)
 348             for node_spec in site_spec['nodes']:
 349                 test_node=TestNode (self,test_site,node_spec)
 350                 test_node.create_boot_cd(options.path)
 351         return True
 352
 353     def initscripts (self, options):
 354         for initscript in self.plc_spec['initscripts']:
 355             utils.show_spec('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
 356             self.server.AddInitScript(self.auth_root(),initscript['initscript_fields'])
 357         return True
 358
 359     def slices (self, options):
 360         return self.do_slices()
 361
 362     def clean_slices (self, options):
 363         return self.do_slices("delete")
 364
 365     def do_slices (self,  action="add"):
 366         for slice in self.plc_spec['slices']:
 367             site_spec = self.locate_site (slice['sitename'])
 368             test_site = TestSite(self,site_spec)
 369             test_slice=TestSlice(self,test_site,slice)
 370             if action != "add":
 371                 utils.header("Deleting slices in site %s"%test_site.name())
 372                 test_slice.delete_slice()
 373             else:
 374                 utils.show_spec("Creating slice",slice)
 375                 test_slice.create_slice()
 376                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
 377         return True
 378
 379     def check_slices(self, options):
 380         for slice_spec in self.plc_spec['slices']:
 381             site_spec = self.locate_site (slice_spec['sitename'])
 382             test_site = TestSite(self,site_spec)
 383             test_slice=TestSlice(self,test_site,slice_spec)
 384             status=test_slice.do_check_slices()
 385             return status
 386
 387     def start_nodes (self, options):
 388         self.kill_all_vmwares()
 389         self.kill_all_qemus()
 390         utils.header("Starting vmware nodes")
 391         for site_spec in self.plc_spec['sites']:
 392             TestSite(self,site_spec).start_nodes (options)
 393         return True
 394
 395     def stop_nodes (self, options):
 396         self.kill_all_vmwares ()
 397         self.kill_all_qemus()
 398         return True
 399
 400     # returns the filename to use for sql dump/restore, using options.dbname if set
 401     def dbfile (self, database, options):
 402         # uses options.dbname if it is found
 403         try:
 404             name=options.dbname
 405             if not isinstance(name,StringTypes):
 406                 raise Exception
 407         except:
 408             t=datetime.datetime.now()
 409             d=t.date()
 410             name=str(d)
 411         return "/root/%s-%s.sql"%(database,name)
 412
 413     def db_dump(self, options):
 414
 415         dump=self.dbfile("planetab4",options)
 416         self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
 417         utils.header('Dumped planetlab4 database in %s'%dump)
 418         return True
 419
 420     def db_restore(self, options):
 421         dump=self.dbfile("planetab4",options)
 422         ##stop httpd service
 423         self.run_in_guest('service httpd stop')
 424         # xxx - need another wrapper
 425         self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
 426         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
 427         self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
 428         ##starting httpd service
 429         self.run_in_guest('service httpd start')
 430
 431         utils.header('Database restored from ' + dump)