From: Sandrine Avakian Date: Tue, 21 Aug 2012 14:28:26 +0000 (+0200) Subject: Fixed bug when sfi.py resources . X-Git-Tag: sfa-2.1-24~3^2~96 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=18f8aa461fee5be2dff2562f3d199804eb140026;p=sfa.git Fixed bug when sfi.py resources . Now showing resources in the slice, as well as the leases if specified. --- diff --git a/sfa/senslab/slabaggregate.py b/sfa/senslab/slabaggregate.py index 2f98c77b..6ade1477 100644 --- a/sfa/senslab/slabaggregate.py +++ b/sfa/senslab/slabaggregate.py @@ -76,20 +76,26 @@ class SlabAggregate: # sort slivers by node id , if there is a job #and therfore, node allocated to this slice for sfa_slice in slices: - if sfa_slice['oar_job_id'] is not -1: - try: + try: - for node_id in sfa_slice['node_ids']: - #node_id = self.driver.root_auth + '.' + node_id - sliver = Sliver({'sliver_id': urn_to_sliver_id(slice_urn, \ - sfa_slice['record_id_slice'], node_id), - 'name': sfa_slice['slice_hrn'], - 'type': 'slab-node', - 'tags': []}) - slivers[node_id] = sliver - except KeyError: - logger.log_exc("SLABAGGREGATE \t \ - get_slice_and_slivers KeyError ") + for node_id in sfa_slice['node_ids']: + #node_id = self.driver.root_auth + '.' + node_id + sliver = Sliver({'sliver_id': urn_to_sliver_id(slice_urn, \ + sfa_slice['record_id_slice'], node_id), + 'name': sfa_slice['slice_hrn'], + 'type': 'slab-node', + 'tags': []}) + slivers[node_id] = sliver + except KeyError: + logger.log_exc("SLABAGGREGATE \t \ + get_slice_and_slivers KeyError ") + + #Add default sliver attribute : + #connection information for senslab + tmp = sfa_slice['slice_hrn'].split('.') + ldap_username = tmp[1].split('_')[0] + vmaddr = 'ssh ' + ldap_username + '@grenoble.senslab.info' + slivers['default_sliver'] = {'vm': vmaddr , 'login': ldap_username} ## sort sliver attributes by node id ##tags = self.driver.GetSliceTags({'slice_tag_id': slice['slice_tag_ids']}) ##for tag in tags: @@ -102,7 +108,7 @@ class SlabAggregate: ##slivers[tag['node_id']]['tags'].append(tag) logger.debug("SLABAGGREGATE api get_slice_and_slivers slivers %s "\ %(slivers)) - return (sfa_slice, slivers) + return (slices, slivers) @@ -114,6 +120,9 @@ class SlabAggregate: # but what is the role of the slivers parameter ? # So i assume that slice['node_ids'] will be the same as slivers for us #filter_dict = {} + #if slice_xrn: + #if not slices or not slices['node_ids']: + #return ([],[]) tags_filter = {} # get the granularity in second for the reservation system @@ -157,95 +166,94 @@ class SlabAggregate: #interfaces = self.get_interfaces({'interface_id':interface_ids}) # get tags #node_tags = self.get_node_tags(tags_filter) - - - reserved_nodes = self.driver.GetNodesCurrentlyInUse() + + #if slices, this means we got to list all the nodes given to this slice + # Make a list of all the nodes in the slice before getting their attributes rspec_nodes = [] + slice_nodes_list = [] + if slices: + for one_slice in slices: + for node_id in one_slice['node_ids']: + slice_nodes_list.append(node_id) + + reserved_nodes = self.driver.GetNodesCurrentlyInUse() + logger.debug("SLABAGGREGATE api get_rspec slice_nodes_list %s "\ + %(slice_nodes_list)) for node in nodes: # skip whitelisted nodes #if node['slice_ids_whitelist']: #if not slice or slice['slice_id'] not in node['slice_ids_whitelist']: #continue #rspec_node = Node() - rspec_node = SlabNode() - # xxx how to retrieve site['login_base'] - #site_id=node['site_id'] - #site=sites_dict[site_id] - rspec_node['mobile'] = node['mobile'] - rspec_node['archi'] = node['archi'] - rspec_node['radio'] = node['radio'] - - slab_xrn = slab_xrn_object(self.driver.root_auth, node['hostname']) - rspec_node['component_id'] = slab_xrn.urn - rspec_node['component_name'] = node['hostname'] - rspec_node['component_manager_id'] = \ - hrn_to_urn(self.driver.root_auth, 'authority+sa') + logger.debug("SLABAGGREGATE api get_rspec node %s "\ + %(node)) + if slice_nodes_list == [] or node['hostname'] in slice_nodes_list: + + rspec_node = SlabNode() + # xxx how to retrieve site['login_base'] + #site_id=node['site_id'] + #site=sites_dict[site_id] + rspec_node['mobile'] = node['mobile'] + rspec_node['archi'] = node['archi'] + rspec_node['radio'] = node['radio'] + + slab_xrn = slab_xrn_object(self.driver.root_auth, node['hostname']) + rspec_node['component_id'] = slab_xrn.urn + rspec_node['component_name'] = node['hostname'] + rspec_node['component_manager_id'] = \ + hrn_to_urn(self.driver.root_auth, 'authority+sa') + + # Senslab's nodes are federated : there is only one authority + # for all Senslab sites, registered in SFA. + # Removing the part including the site in authority_id SA 27/07/12 + rspec_node['authority_id'] = rspec_node['component_manager_id'] + + # do not include boot state ( element) in the manifest rspec + + + rspec_node['boot_state'] = node['boot_state'] + if node['hostname'] in reserved_nodes: + rspec_node['boot_state'] = "Reserved" + rspec_node['exclusive'] = 'True' + rspec_node['hardware_types'] = [HardwareType({'name': 'slab-node'})] + + # only doing this because protogeni rspec needs + # to advertise available initscripts + #rspec_node['pl_initscripts'] = None + # add site/interface info to nodes. + # assumes that sites, interfaces and tags have already been prepared. + #site = sites_dict[node['site_id']] + location = Location({'country':'France'}) + rspec_node['location'] = location - # Senslab's nodes are federated : there is only one authority - # for all Senslab sites, registered in SFA. - # Removing the part including the site in authority_id SA 27/07/12 - rspec_node['authority_id'] = rspec_node['component_manager_id'] - - # do not include boot state ( element) in the manifest rspec - #if not slice: - # rspec_node['boot_state'] = node['boot_state'] - # if node['hostname'] in reserved_nodes: - # rspec_node['boot_state'] = "Reserved" - rspec_node['boot_state'] = node['boot_state'] - if node['hostname'] in reserved_nodes: - rspec_node['boot_state'] = "Reserved" - rspec_node['exclusive'] = 'True' - rspec_node['hardware_types'] = [HardwareType({'name': 'slab-node'})] - - # only doing this because protogeni rspec needs - # to advertise available initscripts - #rspec_node['pl_initscripts'] = None - # add site/interface info to nodes. - # assumes that sites, interfaces and tags have already been prepared. - #site = sites_dict[node['site_id']] - location = Location({'country':'France'}) - rspec_node['location'] = location - - - position = SlabPosition() - for field in position : - try: - position[field] = node[field] - except KeyError, error : - logger.log_exc("SLABAGGREGATE\t get_rspec position %s "%(error)) - - rspec_node['position'] = position - #rspec_node['interfaces'] = [] - #if_count=0 - #for if_id in node['interface_ids']: - #interface = Interface(interfaces[if_id]) - #interface['ipv4'] = interface['ip'] - #interface['component_id'] = PlXrn(auth=self.driver.hrn, - #interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn() - # interfaces in the manifest need a client id - #if slice: - #interface['client_id'] = "%s:%s" % (node['node_id'], if_id) - #rspec_node['interfaces'].append(interface) - #if_count+=1 - - #tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']] - # Granularity - granularity = Granularity({'grain': grain}) - rspec_node['granularity'] = granularity - rspec_node['tags'] = [] - if node['hostname'] in slivers: - # add sliver info - sliver = slivers[node['hostname']] - rspec_node['sliver_id'] = sliver['sliver_id'] - rspec_node['client_id'] = node['hostname'] - rspec_node['slivers'] = [sliver] - - # slivers always provide the ssh service - #login = Login({'authentication': 'ssh-keys', 'hostname': node['hostname'], 'port':'22', 'username': sliver['name']}) - #service = Services({'login': login}) - #rspec_node['services'] = [service] - rspec_nodes.append(rspec_node) + position = SlabPosition() + for field in position : + try: + position[field] = node[field] + except KeyError, error : + logger.log_exc("SLABAGGREGATE\t get_rspec position %s "%(error)) + + rspec_node['position'] = position + #rspec_node['interfaces'] = [] + + #tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']] + # Granularity + granularity = Granularity({'grain': grain}) + rspec_node['granularity'] = granularity + rspec_node['tags'] = [] + if node['hostname'] in slivers: + # add sliver info + sliver = slivers[node['hostname']] + rspec_node['sliver_id'] = sliver['sliver_id'] + rspec_node['client_id'] = node['hostname'] + rspec_node['slivers'] = [sliver] + + # slivers always provide the ssh service + #login = Login({'authentication': 'ssh-keys', 'hostname': node['hostname'], 'port':'22', 'username': sliver['name']}) + #service = Services({'login': login}) + #rspec_node['services'] = [service] + rspec_nodes.append(rspec_node) return (rspec_nodes) @@ -333,12 +341,15 @@ class SlabAggregate: try: lease_option = options['list_leases'] except KeyError: - return + #If no options are specified, at least print the resources + if slice_xrn : + lease_option = 'all' + pass - if lease_option == 'resources': + if lease_option in ['all', 'resources']: #if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'leases': nodes = self.get_nodes(slices, slivers) - #In case creating a job slice _xrn is not set to None + #In case creating a job, slice_xrn is not set to None rspec.version.add_nodes(nodes) if slice_xrn : #Get user associated with this slice @@ -346,24 +357,24 @@ class SlabAggregate: #slices['record_id_user']).first() #ldap_username = (user.hrn).split('.')[1] - ldap_username = slices['slice_hrn'] + + + #for one_slice in slices : + ldap_username = slices[0]['slice_hrn'] tmp = ldap_username.split('.') ldap_username = tmp[1].split('_')[0] - logger.debug("SlabAggregate \tget_rspec **** \ - ldap_username %s rspec.version %s\r\n" %(ldap_username, rspec.version)) + if version.type == "Slab": rspec.version.add_connection_information(ldap_username) - default_sliver = slivers.get(None, []) + default_sliver = slivers.get('default_sliver', []) if default_sliver: - default_sliver_attribs = default_sliver.get('tags', []) + #default_sliver_attribs = default_sliver.get('tags', []) logger.debug("SlabAggregate \tget_rspec **** \ - default_sliver_attribs %s \r\n" %(default_sliver_attribs)) - for attrib in default_dbsessionsliver_attribs: - - - rspec.version.add_default_sliver_attribute(attrib['tagname'], \ - attrib['value']) + default_sliver%s \r\n" %(default_sliver)) + for attrib in default_sliver: + rspec.version.add_default_sliver_attribute(attrib, \ + default_sliver[attrib]) if lease_option in ['all','leases']: #if options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'resources': leases = self.get_leases(slices) diff --git a/sfa/senslab/slabdriver.py b/sfa/senslab/slabdriver.py index a9bfce07..8c193f38 100644 --- a/sfa/senslab/slabdriver.py +++ b/sfa/senslab/slabdriver.py @@ -194,9 +194,9 @@ class SlabDriver(Driver): if sl['oar_job_id'] != [] : #one entry in the dictionnary for each jobid/login, one login #can have multiple jobs running - for oar_jobid in sl['oar_job_id']: - if (login, oar_jobid) not in sfa_slices_dict: - sfa_slices_dict[(login,oar_jobid)] = sl + #for oar_jobid in sl['oar_job_id']: + if (login, sl['oar_job_id']) not in sfa_slices_dict: + sfa_slices_dict[(login,sl['oar_job_id'])] = sl for lease in oar_leases_list: if (lease['user'], lease['lease_id']) not in oar_leases_dict: @@ -246,8 +246,8 @@ class SlabDriver(Driver): # parse rspec rspec = RSpec(rspec_string) - logger.debug("SLABDRIVER.PY \tcreate_sliver \trspec.version %s " \ - %(rspec.version)) + logger.debug("SLABDRIVER.PY \t create_sliver \tr spec.version %s slice_record %s " \ + %(rspec.version,slice_record)) self.synchronize_oar_and_slice_table(slice_hrn) # ensure site record exists? @@ -880,7 +880,7 @@ class SlabDriver(Driver): slicerec = {} slicerec_dict = {} authorized_filter_types_list = ['slice_hrn', 'record_id_user'] - + slicerec_dictlist = [] if slice_filter_type in authorized_filter_types_list: #Get list of slices based on the slice hrn if slice_filter_type == 'slice_hrn': @@ -902,9 +902,9 @@ class SlabDriver(Driver): return [] #slicerec_dictlist = [] - slicerec_dict = slicerec.dump_sqlalchemyobj_to_dict() + fixed_slicerec_dict = slicerec.dump_sqlalchemyobj_to_dict() if login is None : - login = slicerec_dict['slice_hrn'].split(".")[1].split("_")[0] + login = fixed_slicerec_dict['slice_hrn'].split(".")[1].split("_")[0] #for record in slicerec: #slicerec_dictlist.append(record.dump_sqlalchemyobj_to_dict()) @@ -912,48 +912,56 @@ class SlabDriver(Driver): #login = slicerec_dictlist[0]['slice_hrn'].split(".")[1].split("_")[0] #One slice can have multiple jobs - sqljob_list = slab_dbsession.query(JobSenslab).filter_by( slice_hrn=slicerec_dict['slice_hrn']).all() + sqljob_list = slab_dbsession.query(JobSenslab).filter_by( slice_hrn=fixed_slicerec_dict['slice_hrn']).all() job_list = [] for job in sqljob_list: job_list.append(job.dump_sqlalchemyobj_to_dict()) - logger.debug("\r\n SLABDRIVER \tGetSlices login %s \ + logger.debug(" SLABDRIVER \tGetSlices login %s \ slice record %s" \ - %(login, slicerec_dict)) + %(login, fixed_slicerec_dict)) - #Several jobs for one slice - slicerec_dict['oar_job_id'] = [] - for job in job_list : - #if slicerec_dict['oar_job_id'] is not -1: + #Several jobs for one slice + #TODO : Modify to make a diff with jobs not terminated = 1 OAR request SA 20/08/12 + for job in job_list : + slicerec_dict = {} + slicerec_dict['oar_job_id'] = [] + #Check with OAR the status of the job if a job id is in #the slice record rslt = self.GetJobsResources(job['oar_job_id'], \ username = login) - logger.debug("SLABDRIVER.PY \tGetSlices rslt fromn GetJobsResources %s"\ - %(rslt)) + logger.debug("SLABDRIVER.PY \tGetSlices job %s \trslt fromn GetJobsResources %s \r\n"\ + %(job, rslt)) if rslt : - slicerec_dict['oar_job_id'].append(job['oar_job_id']) + slicerec_dict['oar_job_id']= job['oar_job_id'] slicerec_dict.update(rslt) + slicerec_dict.update(fixed_slicerec_dict) slicerec_dict.update({'hrn':\ - str(slicerec_dict['slice_hrn'])}) + str(fixed_slicerec_dict['slice_hrn'])}) + + #If GetJobsResources is empty, this means the job is #now in the 'Terminated' state #Update the slice record else : self.db.delete_job(slice_filter, job['oar_job_id']) slicerec_dict.\ - update({'hrn':str(slicerec_dict['slice_hrn'])}) - + update({'hrn':str(fixed_slicerec_dict['slice_hrn'])}) + try: slicerec_dict['node_ids'] = job['node_list'] except KeyError: pass - - logger.debug("SLABDRIVER.PY \tGetSlices RETURN slicerec_dict %s"\ - %(slicerec_dict)) + + slicerec_dictlist.append(slicerec_dict) + logger.debug("SLABDRIVER.PY \tGetSlices slicerec_dict %s slicerec_dictlist %s" %(slicerec_dict, slicerec_dictlist)) + + logger.debug("SLABDRIVER.PY \tGetSlices RETURN slicerec_dictlist %s"\ + %(slicerec_dictlist)) - return [slicerec_dict] + return slicerec_dictlist else: diff --git a/sfa/senslab/slabslices.py b/sfa/senslab/slabslices.py index 0eb91603..1b8859af 100644 --- a/sfa/senslab/slabslices.py +++ b/sfa/senslab/slabslices.py @@ -146,8 +146,9 @@ class SlabSlices: peer): - #First get the list of current leases from OAR - leases = self.driver.GetLeases({'name':sfa_slice['name']}, ['lease_id']) + #First get the list of current leases from OAR + leases = self.driver.GetLeases({'name':sfa_slice['name']}) + #leases = self.driver.GetLeases({'name':sfa_slice['name']}, ['lease_id']) if leases : current_leases = [lease['lease_id'] for lease in leases] #Deleted leases are the ones with lease id not declared in the Rspec @@ -372,6 +373,7 @@ class SlabSlices: def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, \ options={}): + #TODO SA 21/08/12 verify_persons Needs review users_by_id = {} users_by_hrn = {} users_dict = {} @@ -399,76 +401,56 @@ class SlabSlices: existing_user_ids = [] existing_user_hrns = [] existing_users = [] - #Check if user is in LDAP using its hrn. + #Check if user is in Senslab LDAP using its hrn. #Assuming Senslab is centralised : one LDAP for all sites, # user_id unknown from LDAP - # LDAP does not provide users id, therfore we rely on hrns + # LDAP does not provide users id, therefore we rely on hrns + # If the hrn is not a senslab hrn, the user may not be in LDAP. if users_by_hrn: #Construct the list of filters for GetPersons filter_user = [] for hrn in users_by_hrn: - #filter_user.append ( {'hrn':hrn}) filter_user.append (users_by_hrn[hrn]) logger.debug(" SLABSLICE.PY \tverify_person filter_user %s " \ %(filter_user)) existing_users = self.driver.GetPersons(filter_user) - #existing_users = self.driver.GetPersons({'hrn': \ - #users_by_hrn.keys()}) - #existing_users = self.driver.GetPersons({'hrn': \ - #users_by_hrn.keys()}, \ - #['hrn','pkey']) if existing_users: for user in existing_users : - #for k in users_dict[user['hrn']] : - existing_user_hrns.append(users_dict[user['hrn']]['hrn']) existing_user_ids.\ append(users_dict[user['hrn']]['person_id']) #User from another federated site , - #does not have a senslab account yet? - #or have multiple SFA accounts + #does not have a senslab account yet? + #Check in the LDAP if we know email, + #maybe he has multiple SFA accounts = multiple hrns. #Check before adding them to LDAP - else: - + req = 'mail=' if isinstance(users, list): - ldap_reslt = self.driver.ldap.LdapSearch(users[0]) + req += users[0]['email'] else: - ldap_reslt = self.driver.ldap.LdapSearch(users) + req += users['email'] + + ldap_reslt = self.driver.ldap.LdapSearch(req) if ldap_reslt: - existing_users = ldap_reslt[0] - #TODO : DEBUG user undefined ? SA 14/08/12 - existing_user_hrns.append(users_dict[user['hrn']]['hrn']) - existing_user_ids.\ - append(users_dict[user['hrn']]['person_id']) + logger.debug(" SLABSLICE.PY \tverify_person users \ + USER already in Senslab \t ldap_reslt %s "%( ldap_reslt)) + existing_users.append(ldap_reslt[1]) + else: #User not existing in LDAP - + #TODO SA 21/08/12 raise something to add user or add it auto ? logger.debug(" SLABSLICE.PY \tverify_person users \ - not in ldap ... %s \r\n \t ldap_reslt %s " \ + not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t ldap_reslt %s " \ %(users, ldap_reslt)) - - - # requested slice users + requested_user_ids = users_by_id.keys() requested_user_hrns = users_by_hrn.keys() logger.debug("SLABSLICE.PY \tverify_person requested_user_ids %s \ user_by_hrn %s " %(requested_user_ids, users_by_hrn)) - # existing slice users - - #existing_slice_users_filter = {'hrn': slice_record['PI'][0]} - #logger.debug(" SLABSLICE.PY \tverify_person requested_user_ids %s \ - #existing_slice_users_filter %s slice_record %s" %(requested_user_ids,\ - #existing_slice_users_filter,slice_record)) - - #existing_slice_users = \ - #self.driver.GetPersons([existing_slice_users_filter]) - #existing_slice_users = \ - #self.driver.GetPersons(existing_slice_users_filter, \ - #['hrn','pkey']) - #logger.debug("SLABSLICE.PY \tverify_person existing_slice_users %s " \ - #%(existing_slice_users)) + + #Check that the user of the slice in the slice record #matches the existing users try: @@ -503,8 +485,10 @@ class SlabSlices: added_user = users_dict[added_user_hrn] #hrn, type = urn_to_hrn(added_user['urn']) person = { - 'first_name': added_user.get('first_name', hrn), - 'last_name': added_user.get('last_name', hrn), + #'first_name': added_user.get('first_name', hrn), + #'last_name': added_user.get('last_name', hrn), + 'first_name': added_user['first_name'], + 'last_name': added_user['last_name'], 'person_id': added_user['person_id'], 'peer_person_id': None, 'keys': [],