merge from onelab
authorThierry Parmentelat <thierry.parmentelat@sophia.inria.fr>
Thu, 25 Oct 2007 19:22:54 +0000 (19:22 +0000)
committerThierry Parmentelat <thierry.parmentelat@sophia.inria.fr>
Thu, 25 Oct 2007 19:22:54 +0000 (19:22 +0000)
plctest/TestConfig.py
plctest/TestMain.py
plctest/TestNode.py
plctest/TestPlc.py
plctest/TestSite.py

index 4dcb61c..67357a2 100644 (file)
@@ -1,5 +1,4 @@
 #definition of all structure used by the test_setup.py script
-
 site1_nodes = {
 'node1' :  {'hostname': 'test1.one-lab.org',
             'boot_state':'inst',
@@ -32,26 +31,14 @@ site1_nodes = {
               
               },
 }
-
-site_spec1 = {
-'site_fields' : {'name':'testsite',
-                'login_base':'ts',
-                'abbreviated_name':'PLanettest',
-                'max_slices':100,
-                'url':'http://onelab-test.inria.fr',
-                },
-'site_address' : {'line1':'route des lucioles',
-                 'city':'sophia',
-                 'state':'fr',
-                 'postalcode':'06600',
-                 'country':'france',
-                 },
+site_users= {
 'pi_spec' : {'first_name':'PI',
               'last_name':'PI',
               'enabled':'True',
               'email':'fake-pi1@one-lab.org',
               'password':'testpi',
               'roles':['pi'],
+               'auth_meth':'pi',
               },
 'tech_spec' : {'first_name':'Tech',
                'last_name':'Tech',
@@ -59,6 +46,7 @@ site_spec1 = {
                 'email':'fake-tech1@one-lab.org',
                 'password':'testtech',
                 'roles':['tech'],
+                 'auth_meth':'tech',
                 },
 'user_spec' : {'first_name':'User',
                 'last_name':'User',
@@ -66,6 +54,7 @@ site_spec1 = {
                 'email':'fake-user1@one-lab.org',
                 'password':'testuser',
                 'roles':['user'],
+                 'auth_meth':'user',
                 },
 'tech_user_spec' : {'first_name':'UserTech',
                 'last_name':'UserTech',
@@ -73,6 +62,7 @@ site_spec1 = {
                 'email':'fake-tech2@one-lab.org',
                 'password':'testusertech',
                 'roles':['tech','user'],
+                 'auth_meth':'techuser',
                  },
 'pi_tech_spec' : {'first_name':'PiTech',
                 'last_name':'PiTech',
@@ -80,8 +70,23 @@ site_spec1 = {
                 'email':'fake-pi2@one-lab.org',
                 'password':'testusertech',
                 'roles':['pi','tech'],
+                 'auth_meth':'pitech',
                   },
-
+}
+site_spec1 = {
+'site_fields' : {'name':'testsite',
+                'login_base':'ts',
+                'abbreviated_name':'PLanettest',
+                'max_slices':100,
+                'url':'http://onelab-test.inria.fr',
+                },
+'site_address' : {'line1':'route des lucioles',
+                 'city':'sophia',
+                 'state':'fr',
+                 'postalcode':'06600',
+                 'country':'france',
+                 },
+'users': [ site_users['pi_spec'], site_users['tech_spec'], site_users['user_spec'],site_users['tech_user_spec'],site_users['pi_tech_spec']],
 'nodes' :  [ site1_nodes['node1'], site1_nodes['node2']],
 }
 
@@ -97,7 +102,6 @@ plc_spec1 =  {
     'PLC_MAIL_ENABLED':'true',
     'PLC_MAIL_SUPPORT_ADDRESS' : 'mohamed-amine.chaoui@sophia.inria.fr',
     'PLC_DB_HOST' : 'onelab-test.inria.fr',
-    'PLC_API_DEBUG':'true',
     'PLC_API_HOST' : 'onelab-test.inria.fr',
     'PLC_WWW_HOST' : 'onelab-test.inria.fr',
     'PLC_BOOT_HOST' : 'onelab-test.inria.fr',
@@ -105,18 +109,23 @@ plc_spec1 =  {
     'PLC_NET_DNS2' : '138.96.0.11',
     'sites' : site_specs,
     }
-
 plc_specs = [ plc_spec1 ]
 
 key={'key_type':'ssh',
      'key':'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA4jNj8yT9ieEc6nSJz/ESu4fui9WrJ2y/MCfqIZ5WcdVKhBFUYyIenmUaeTduMcSqvoYRQ4QnFR1BFdLG8XR9D6FWZ5zTKUgpkew22EVNeqai4IXeWYKyt1Qf3ehaz9E3o1PG/bmQNIM6aQay6TD1Y4lqXI+eTVXVQev4K2fixySjFQpp9RB4UHbeA8c28yoa/cgAYHqCqlvm9uvpGMjgm/Qa4M+ZeO7NdjowfaF/wF4BQIzVFN9YRhvQ/d8WDz84B5Pr0J7pWpaX7EyC4bvdskxl6kmdNIwIRcIe4OcuIiX5Z9oO+7h/chsEVJWF4vqNIYlL9Zvyhnr0hLLhhuk2bw== root@onelab-test.inria.fr'}
 
 
-slice_spec={'name':'ts_slicetest1',
+slice1_spec={
+'slice_spec':{'name':'ts_slicetest1',
               'instantiation':'plc-instantiated',
               'url':'http://foo@ffo.com',
               'description':'testslice the first slice for the site testsite',
               'max_nodes':1000
-              }
+              },
+'slice_users' : [ site_users['pi_spec'], site_users['tech_spec'],site_users['tech_user_spec']],
+'slice_nodes' : [ site1_nodes['node1'], site1_nodes['node2'] ],
+}
+slices_specs= [slice1_spec]
+
+
 
-file_name='%d%s.iso'
index 93b0f8c..38c0f5f 100755 (executable)
@@ -2,12 +2,12 @@
 
 import os, sys, time
 from optparse import OptionParser
-
 from TestPlc import TestPlc
 from TestSite import TestSite
 from TestNode import TestNode
 import TestConfig
 import threading
+
 class TestMain:
 
     subversion_id = "$Id: TestMain.py 767 2007-08-06 08:32:04Z amine $"
@@ -28,25 +28,27 @@ class TestMain:
             #exporting Display
             parser.add_option("-d","--display", action="store", dest="Xterm", default='bellami:0.0',
                               help="export the display on the mentionneted one")
-       
         
             (self.options, self.args) = parser.parse_args()
 
             display=''
             url=''
-            
-           
+            test_plcs=[]
+            test_nodes=[]
+            pids=[]
+            timset=time.strftime("%H:%M:%S", time.localtime())
+            #test the existence of the URL
             if (len (self.args)):
                 url=self.args[0]
                 print 'the myplc url is ',url
             else:
                 print "PLease introduce a right URL for the myplc instal"
                 sys.exit(1)
-
+            #check where to display Virtual machines
             if (self.options.Xterm):
                 display=self.options.Xterm
                 print 'the display is', display
-
+            #the debug option 
             if (self.options.debug):
                 file=self.path+'/'+self.options.debug+'/My_Virtual_Machine.vmx'
                 if os.path.exists(file):
@@ -57,13 +59,6 @@ class TestMain:
                 else:
                     print "no way to find the virtual file"
                     sys.exit(1)
-                            
-
-            
-            test_plcs=[]
-            test_nodes=[]
-            pids=[]
-            timset=time.strftime("%H:%M:%S", time.localtime())
             
             for plc_spec in TestConfig.plc_specs:
                 print '========>Creating plc at '+timset+':',plc_spec
@@ -77,17 +72,16 @@ class TestMain:
                     test_plc.config_plc(plc_spec)
                 else :
                     print "========>PLease insert a valid url for the myplc install"
-        
+                ##create all the sites under the new plc,and then populate them with
+                ##nodes,persons and slices
                 for site_spec in plc_spec['sites']:
                     print '========>Creating site at '+timset+ ':',site_spec
                     test_site = test_plc.init_site(site_spec)
-                    
                     for node_spec in site_spec['nodes']:
                         print '========>Creating node at  '+ timset+' :',node_spec
                         test_nodes.append(node_spec)
                         test_node = test_plc.init_node(test_site,node_spec,self.path)
-                        
-                test_node.create_slice ("pi",test_nodes)
+                test_node.create_slice ("pi")
                 print 'Runing Checkers and Vmwares for Site nodes at :',timset
                 test_site.run_vmware(test_nodes,display)
                 if(test_site.node_check_status(test_nodes,True)):
@@ -95,11 +89,9 @@ class TestMain:
                     test_site.slice_access(test_nodes)
                     print "all is alright"
                     return 0
-                    #sys.exit(0)
                 else :
                     print "There is something wrong"
                     sys.exit(1)
-                
         except Exception, e:
             print str(e)
             sys.exit(1)
index 640f710..0ca91ba 100644 (file)
@@ -13,7 +13,7 @@ class TestNode:
        self.node_spec=node_spec
         self.timset=time.strftime("%H:%M:%S", time.localtime())
     def create_node (self,role):
-        auth = self.test_site.anyuser_auth (role+"_spec")
+        auth = self.test_site.anyuser_auth (role)
         filter={'boot_state':'rins'}
         try:
             if (role=='pi' and self.node_spec['owned']=='pi'):
@@ -36,24 +36,24 @@ class TestNode:
         except Exception, e:
                 print str(e)
 
-    def create_slice(self, role, liste_nodes_spec):
-        print '========>Creating slice at :'+self.timset+' : ',TestConfig.slice_spec
-        auth = self.test_site.anyuser_auth (role+"_spec")
+    def create_slice(self, role):
+        auth = self.test_site.anyuser_auth (role)
         liste_hosts=[]
-        for l in liste_nodes_spec :
-            liste_hosts.append(l['hostname'])
+        #for l in liste_nodes_spec :
+        #    liste_hosts.append(l['hostname'])
         try:
-            slice_id=self.test_plc.server.AddSlice(auth, TestConfig.slice_spec)
-            self.test_plc.server.AddSliceToNodes(auth, slice_id, liste_hosts)
-            self.test_plc.server.AddPersonToSlice(auth, self.test_site.site_spec['pi_spec']['email'], slice_id)
-            self.test_plc.server.AddPersonToSlice(auth, self.test_site.site_spec['tech_spec']['email'], slice_id)
-            self.test_plc.server.AddPersonToSlice(auth, self.test_site.site_spec['tech_user_spec']['email'], slice_id)
+            for slicespec in TestConfig.slices_specs :
+                print '========>Creating slice at :'+self.timset+' : ',slicespec
+                slice_id=self.test_plc.server.AddSlice(auth,slicespec['slice_spec'])
+                for sliceuser in slicespec['slice_users']:
+                    self.test_plc.server.AddPersonToSlice(auth, sliceuser['email'], slice_id)##affecting person to the slice
+                for slicenode in slicespec['slice_nodes']:
+                    liste_hosts.append(slicenode['hostname'])
+                self.test_plc.server.AddSliceToNodes(auth, slice_id, liste_hosts)##add slice to the spec nodes
             print 'fin creation slices'
         except Exception, e:
             print str(e)
             sys.exit(1)
-
-
         
     def conffile(self,image,hostname,path):
         try:
@@ -76,25 +76,19 @@ class TestNode:
         except Exception, e:
             print str(e)
 
-
-
     def create_boot_cd(self,node_spec,path):
         try:
-            os.system('mkdir  -p  %s/VirtualFile-%s  &&  cp  %s/My-Virtual-Machine-model/*  %s/VirtualFile-%s'%(path, node_spec['hostname'], path, path, node_spec['hostname']))
-            
+            os.system('mkdir  -p  %s/VirtualFile-%s  &&  cp  %s/My-Virtual-Machine-model/*  %s/VirtualFile-%s'
+                      %(path, node_spec['hostname'], path, path, node_spec['hostname']))
             link1=self.test_plc.server.GetBootMedium(self.test_plc.auth_root(),
                                                      node_spec['hostname'], 'node-iso', '')
             if (link1 == ''):
                 raise Exception, 'boot.iso not found'
-            
             file1=open(path+'/VirtualFile-'+node_spec['hostname']+'/boot_file.iso','w')
             file1.write(base64.b64decode(link1))
-            
-            
             file1.close()
             print '========> boot cd created for :',self.node_spec['hostname']
             self.conffile('boot_file.iso',self.node_spec['hostname'], path) #create 2 conf file for the vmware based
-
         except Exception, e:
             print str(e)
             sys.exit(1)
index 52dd8c5..d183e83 100644 (file)
@@ -43,7 +43,6 @@ class TestPlc:
                      'PLC_MAIL_ENABLED',
                      'PLC_MAIL_SUPPORT_ADDRESS',
                      'PLC_DB_HOST',
-                     'PLC_API_DEBUG',
                      'PLC_API_HOST',
                      'PLC_WWW_HOST',
                      'PLC_BOOT_HOST',
@@ -67,8 +66,6 @@ class TestPlc:
         ##### Clean up the /plc directory
         os.system('set -x; rm -rf  /plc/data')
         print "=======================>Clean up  DONE!"
-
-
         
     def install_plc(self,url):
         print url
@@ -78,10 +75,10 @@ class TestPlc:
     def init_site (self,site_spec):
         test_site = TestSite (self,site_spec)
         test_site.create_site()
-        for key in ['pi_spec','tech_spec','user_spec','tech_user_spec','pi_tech_spec']:
-            test_site.create_user(site_spec[key])
-            test_site.enable_user(site_spec[key])
-            test_site.add_key_user(site_spec[key])            
+        for key in site_spec['users']:
+            test_site.create_user(key)
+            test_site.enable_user(key)
+            test_site.add_key_user(key)            
         return test_site
 
     def init_node (self,test_site,node_spec,path):
@@ -90,10 +87,7 @@ class TestPlc:
         test_node.create_node ("pi")
         test_node.create_node ("tech")
         test_node.create_boot_cd(node_spec,path)
-       
-
         return test_node
-
     
     def db_dump(self):
         
index 2179026..f9d4caa 100644 (file)
@@ -2,7 +2,7 @@ import os
 import sys
 import datetime
 import time
-import TestConfig
+from TestConfig import *
 import xmlrpclib
 
 class TestSite:
@@ -28,7 +28,6 @@ class TestSite:
        return self.site_id()
 
     def create_user (self, user_spec):
-        
         try:
             i=0
             print '========>Adding user at '+self.timset+ ': ',user_spec
@@ -36,21 +35,15 @@ class TestSite:
                                                           user_spec)
             self.test_plc.server.UpdatePerson(self.test_plc.auth_root(),
                                               self.person_id,{'enabled': True})
-            
-            for i in range(0, len(user_spec['roles']) ):
+            for role in user_spec['roles']:
                 self.test_plc.server.AddRoleToPerson(self.test_plc.auth_root(),
-                                                     user_spec['roles'][i],
-                                                     user_spec['email'])
-                i=i+1
-                
+                                                     role,user_spec['email'])
             self.test_plc.server.AddPersonToSite(self.test_plc.auth_root(),
                                                  user_spec['email'],
                                                  self.site_spec['site_fields']['login_base'])
-            
         except Exception,e:
             print str(e)
             
-       
     def enable_user (self, user_spec):
         try:
             persones=self.test_plc.server.GetPersons(self.test_plc.auth_root())
@@ -61,56 +54,40 @@ class TestSite:
                                                       {'enabled': True})
         except Exception,e:
             print str(e)
-
-
+            
     def add_key_user(self,user_spec):
         try:
-            if(user_spec ==self.site_spec['pi_spec']):
-                self.test_plc.server.AddPersonKey(self.pi_auth(), user_spec['email'], TestConfig.key)
-            elif(user_spec ==self.site_spec['tech_spec']):
-                self.test_plc.server.AddPersonKey(self.tech_auth(), user_spec['email'], TestConfig.key)
-            elif (user_spec ==self.site_spec['user_spec']):
-                self.test_plc.server.AddPersonKey(self.user_auth(), user_spec['email'], TestConfig.key)
-            elif (user_spec ==self.site_spec['tech_user_spec']):
-                self.test_plc.server.AddPersonKey(self.tech_user_auth(), user_spec['email'], TestConfig.key)
-            elif (user_spec ==self.site_spec['pi_tech_spec']):
-                self.test_plc.server.AddPersonKey(self.pi_tech_auth(), user_spec['email'], TestConfig.key)
-                
+            auth=""
+            for userspec in self.site_spec['users']:
+                if(user_spec == userspec):
+                    for role in userspec['roles']:
+                        auth=auth+role
+                    print auth
+                    self.test_plc.server.AddPersonKey(self.anyuser_auth(auth),
+                                                      user_spec['email'], key)
         except Exception, e:
             print str(e)
-
             
     def anyuser_auth (self,key):
-       return {'Username':self.site_spec[key]['email'],
-               'AuthMethod':'password',
-               'AuthString':self.site_spec[key]['password'],
-               'Role':self.site_spec[key]['roles'][0],
-               }
+        for person in self.site_spec['users']:
+            if person['auth_meth']== key :
+                return {'Username':person['email'],
+                        'AuthMethod':'password',
+                        'AuthString':person['password'],
+                        'Role':person['roles'][0],
+                        }
 
-    def pi_auth (self):
-       return self.anyuser_auth('pi_spec')
-    def tech_auth (self):
-       return self.anyuser_auth('tech_spec')
-    def user_auth (self):
-       return self.anyuser_auth('user_spec')
-    def tech_user_auth (self):
-       return self.anyuser_auth('tech_user_spec')
-    def pi_tech_auth (self):
-       return self.anyuser_auth('pi_tech_spec')
-    
     def node_check_status(self,liste_nodes,bool):
         try:
             ret_value=True    
             filter=['boot_state']
             bt={'boot_state':'boot'}
+            dbg={'boot_state':'dbg'}
             secondes=15
-
             start_time = datetime.datetime.now() ##geting the current time
             dead_time=datetime.datetime.now()+ datetime.timedelta(minutes=10)##adding 10minutes
-            
             start=time.strftime("%H:%M:%S", time.localtime())
             print "time in the begining  is :",start
-           
             
             for l in liste_nodes :
                 while (bool):
@@ -123,16 +100,18 @@ class TestSite:
                             test_name='\nTest Installation Node hosted: '+l['hostname']
                             self.test_plc.affiche_results(test_name, 'Successful', '')##printing out the result
                             break ##for exsiting and renaming virtual file to just installed
-                        
+                        elif (node_status[0] ==dbg):
+                            test_name='\nTest Installation Node hosted: '+l['hostname']
+                            self.test_plc.affiche_results(test_name, 'En Debug', '')##printing out the result
+                            bool=False
+                            break ##for exsiting and renaming virtual file to just installed
                         elif ( start_time  <= dead_time ) :
                             start_time=datetime.datetime.now()+ datetime.timedelta(minutes=2)
                             time.sleep(secondes)
-                            
                         else: bool=False
                     except OSError ,e :
                         bool=False
                         str(e)
-                              
                 if (bool):
                     print "Node correctly instaled and booted "
                 else :
@@ -140,8 +119,6 @@ class TestSite:
                     ret_value=False
                     test_name='\nTest Installation Node Hosted: ',l['hostname']
                     self.test_plc.affiche_results(test_name, 'Failure', '')##printing out the result
-                
-                
             
             end=time.strftime("%H:%M:%S", time.localtime())
             print "time at the end is :",end  ##converting time to secondes
@@ -158,7 +135,6 @@ class TestSite:
         os.system('pgrep vmplayer | xargs -r kill ')
         os.system('pgrep vmware | xargs -r kill -9')
         os.system('pgrep vmplayer | xargs -r kill -9')
-
         
     def run_vmware(self,liste_nodes,display):
         path=os.path.dirname(sys.argv[0])
@@ -189,7 +165,6 @@ class TestSite:
         except Exception, e:
             print str(e)
 
-
     def slice_access(self,liste_nodes):
         try:
             bool=True
@@ -198,51 +173,51 @@ class TestSite:
             self.delete_known_hosts()
             start_time = datetime.datetime.now()
             dead_time=start_time + datetime.timedelta(minutes=3)##adding 3minutes
-            for l in liste_nodes:
-                timset=time.strftime("%H:%M:%S", time.localtime())
-                while(bool):
-                    print '=========>Try to Restart the Node Manager on %s at %s:'%(l['hostname'],str(timset))
-                    access=os.system('set -x; ssh -i /etc/planetlab/root_ssh_key.rsa  root@%s service nm restart'%l['hostname'])
-                    if (access==0):
-                        print '=========>Node Manager Restarted on %s at %s:'%(l['hostname'],str(timset))
-                        while(bool1):
-                            print '=========>Try to connect to the %s@%s at %s '%(TestConfig.slice_spec['name'],l['hostname'],str(time.strftime("%H:%M:%S", time.localtime())))
-                            Date=os.system('set -x; ssh -i ~/.ssh/slices.rsa %s@%s echo "The Actual Time here is;" date'%(TestConfig.slice_spec['name'],l['hostname']))
-                            if (Date==0):
-                                break
-                            elif ( start_time  <= dead_time ) :
-                                start_time=datetime.datetime.now()+ datetime.timedelta(seconds=30)
-                                time.sleep(secondes)
-                            else:
-                                bool1=False
-                        if(bool1):
-                            print '=========>connected to the '+TestConfig.slice_spec['name']+'@'+l['hostname']+'--->'
-                        else:
-                            print '=========>access to one slice is denied but last chance'
-                            print '=========>Retry to Restart the Node Manager on %s at %s:'%(l['hostname'],str(timset))
-                            access=os.system('set -x; ssh -i /etc/planetlab/root_ssh_key.rsa  root@%s service nm restart'%l['hostname'])
-                            if (access==0):
-                                print '=========>Retry to connect to the %s@%s at %s '%(TestConfig.slice_spec['name'],l['hostname'],str(time.strftime("%H:%M:%S", time.localtime())))
-                                Date=os.system('set -x; ssh -i ~/.ssh/slices.rsa %s@%s echo "The Actual Time here is;" date'%(TestConfig.slice_spec['name'],l['hostname']))
+            for slice in slices_specs:
+                for slicenode in slice['slice_nodes']:
+                    timset=time.strftime("%H:%M:%S", time.localtime())
+                    while(bool):
+                        print '=========>Try to Restart the Node Manager on %s at %s:'%(slicenode['hostname'],str(timset))
+                        access=os.system('set -x; ssh -i /etc/planetlab/root_ssh_key.rsa  root@%s service nm restart'%slicenode['hostname'] )
+                        if (access==0):
+                            print '=========>Node Manager Restarted on %s at %s:'%(slicenode['hostname'] ,str(timset))
+                            while(bool1):
+                                print '=========>Try to connect to the %s@%s at %s '%(slice['slice_spec']['name'],slicenode['hostname'],str(time.strftime("%H:%M:%S", time.localtime())))
+                                Date=os.system('set -x; ssh -i ~/.ssh/slices.rsa %s@%s echo "The Actual Time here is;" date'%(slice['slice_spec']['name'],slicenode['hostname']))
                                 if (Date==0):
-                                    print '=========>connected to the '+TestConfig.slice_spec['name']+'@'+l['hostname']+'--->'
+                                    break
+                                elif ( start_time  <= dead_time ) :
+                                    start_time=datetime.datetime.now()+ datetime.timedelta(seconds=30)
+                                    time.sleep(secondes)
                                 else:
-                                    print '=========>the Access is finaly denied'
-                                    sys.exit(1)
-                            else :"=========>Last try failed"
-                        break
-                    elif ( start_time  <= dead_time ) :
-                        start_time=datetime.datetime.now()+ datetime.timedelta(minutes=1)
-                        time.sleep(secondes)
-                    else:
-                        bool=False
+                                    bool1=False
+                            if(bool1):
+                                print '=========>connected to the '+slice['slice_spec']['name']+'@'+slicenode['hostname'] +'--->'
+                            else:
+                                print '=========>access to one slice is denied but last chance'
+                                print '=========>Retry to Restart the Node Manager on %s at %s:'%(slicenode['hostname'],str(timset))
+                                access=os.system('set -x; ssh -i /etc/planetlab/root_ssh_key.rsa  root@%s service nm restart'%slicenode['hostname'] )
+                                if (access==0):
+                                    print '=========>Retry to connect to the %s@%s at %s '%(slice['slice_spec']['name'],slicenode['hostname'],str(time.strftime("%H:%M:%S", time.localtime())))
+                                    Date=os.system('set -x; ssh -i ~/.ssh/slices.rsa %s@%s echo "The Actual Time here is;" date'%(slice['slice_spec']['name'],slicenode['hostname'] ))
+                                    if (Date==0):
+                                        print '=========>connected to the '+slice['slice_spec']['name']+'@'+slicenode['hostname']+'--->'
+                                    else:
+                                        print '=========>the Access is finaly denied'
+                                        sys.exit(1)
+                                else :"=========>Last try failed"
+                            break
+                        elif ( start_time  <= dead_time ) :
+                            start_time=datetime.datetime.now()+ datetime.timedelta(minutes=1)
+                            time.sleep(secondes)
+                        else:
+                            bool=False
                                 
-                if (not bool):
-                    print 'Node manager problems'
-                    sys.exit(1)
-                        
+                    if (not bool):
+                        print 'Node manager problems'
+                        sys.exit(1)
                     
         except Exception, e:
             print str(e)
             sys.exit(1)
-
+