From: Sandrine Avakian Date: Thu, 26 Jan 2012 16:09:59 +0000 (+0100) Subject: Moved computation of hrn on the OARrestapi level. X-Git-Tag: sfa-2.1-24~3^2~224 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=c3d687e5065631784ea06698c56f0e53d37a304c;p=sfa.git Moved computation of hrn on the OARrestapi level. Querying slice resources ok but considers that the nodes in the slice are local nodes ( not from a federated senslab platform). This is due to missing information in node hostnames in OAR. TODO : fix parse_filter in case of dict filter. --- diff --git a/sfa/senslab/OARrestapi.py b/sfa/senslab/OARrestapi.py index 758f0b75..cc0fa0b2 100644 --- a/sfa/senslab/OARrestapi.py +++ b/sfa/senslab/OARrestapi.py @@ -6,8 +6,9 @@ from sfa.senslab.parsing import * from sfa.senslab.SenslabImportUsers import * import urllib import urllib2 - - +from sfa.util.config import Config +from sfa.util.plxrn import PlXrn +from sfa.util.xrn import hrn_to_urn, get_authority,Xrn,get_leaf #OARIP='10.127.255.254' OARIP='192.168.0.109' @@ -47,7 +48,8 @@ class OARrestapi: self.oarserver['port'] = 80 self.oarserver['uri'] = None self.oarserver['postformat'] = 'json' - self.parser = OARGETParser(self) + self.parser = OARGETParser(self) + def GETRequestToOARRestAPI(self, request, strval=None ): self.oarserver['uri'] = OARrequests_get_uri_dict[request] @@ -282,7 +284,9 @@ class OARGETParser: else: pass - + + def hostname_to_hrn(self, root_auth, login_base, hostname): + return PlXrn(auth=root_auth,hostname=login_base+'_'+hostname).get_hrn() #Retourne liste de dictionnaires contenant attributs des sites def ParseSites(self): nodes_per_site = {} @@ -290,6 +294,7 @@ class OARGETParser: # Create a list of nodes per site_id for node_id in self.node_dictlist.keys(): node = self.node_dictlist[node_id] + if node['site_login_base'] not in nodes_per_site.keys(): nodes_per_site[node['site_login_base']] = [] nodes_per_site[node['site_login_base']].append(node['node_id']) @@ -300,6 +305,11 @@ class OARGETParser: # and value is a dictionary of properties, including the list of the node_ids for node_id in self.node_dictlist.keys(): node = self.node_dictlist[node_id] + node.update({'hrn':self.hostname_to_hrn(self.interface_hrn, node['site_login_base'],node['hostname'])}) + #node['hrn'] = self.hostname_to_hrn(self.interface_hrn, node['site_login_base'],node['hostname']) + self.node_dictlist.update({node_id:node}) + if node_id is 1: + print>>sys.stderr, " \r\n \r\n \t \t\t\t OARESTAPI Parse Sites self.node_dictlist %s " %(self.node_dictlist) if node['site_login_base'] not in self.site_dict.keys(): self.site_dict[node['site_login_base']] = [('login_base', node['site_login_base']),\ ('node_ids',nodes_per_site[node['site_login_base']]),\ @@ -330,6 +340,8 @@ class OARGETParser: def __init__(self, srv ): self.version_json_dict= { 'api_version' : None , 'apilib_version' :None, 'api_timezone': None, 'api_timestamp': None, 'oar_version': None ,} + self.config = Config() + self.interface_hrn = self.config.SFA_INTERFACE_HRN self.timezone_json_dict = { 'timezone': None, 'api_timestamp': None, } self.jobs_json_dict = { 'total' : None, 'links' : [] , 'offset':None , 'items' : [] , } self.jobs_table_json_dict = self.jobs_json_dict diff --git a/sfa/senslab/slab-import.py b/sfa/senslab/slab-import.py index 37398a8f..888dcccc 100644 --- a/sfa/senslab/slab-import.py +++ b/sfa/senslab/slab-import.py @@ -114,7 +114,7 @@ def import_node(hrn, node): node_record['date_created'] = int(time.mktime(extime.timetuple())) existing_records = table.find({'hrn': hrn, 'type': 'node', 'pointer': node['node_id']}) if not existing_records: - print>>sys.stderr, " \r\n \t slab-import : node record %s inserted" %(node['node_id']) + print>>sys.stderr, " \r\n \t slab-import : node record[hrn]%s inserted" %(node['hrn']) table.insert(node_record) else: existing_record = existing_records[0] @@ -232,7 +232,8 @@ def main(): for node in nodes_dict: # Sandrine # A changer pour l utilisation du nouveau OAR de prod, le site etant contenu dans le hostname - hrn = hostname_to_hrn( root_auth,node['site_login_base'], node['hostname']) + hrn = node['hrn'] + #hrn = hostname_to_hrn( root_auth,node['site_login_base'], node['hostname']) if hrn not in existing_hrns or \ (hrn, 'node') not in existing_records: import_node(hrn, node) diff --git a/sfa/senslab/slabaggregate.py b/sfa/senslab/slabaggregate.py index 48790435..fb4e746b 100644 --- a/sfa/senslab/slabaggregate.py +++ b/sfa/senslab/slabaggregate.py @@ -77,8 +77,8 @@ class SlabAggregate: if slice['oar_job_id'] is not -1: try: - for node_id in slice['assigned_network_address']: - node_id = self.driver.root_auth + '.' + node_id + for node_id in slice['node_ids']: + #node_id = self.driver.root_auth + '.' + node_id sliver = Sliver({'sliver_id': urn_to_sliver_id(slice_urn, slice['record_id_slice'], node_id), 'name': slice['slice_hrn'], 'type': 'slab-vm', @@ -96,100 +96,143 @@ class SlabAggregate: ##'tags': []}) ##slivers[tag['node_id']] = sliver ##slivers[tag['node_id']]['tags'].append(tag) - + print >>sys.stderr,"\r\n \r\n \t\t_____________ Slabaggregate api get_slice_and_slivers slivers %s " %(slivers) return (slice, slivers) def get_nodes(self,slice=None,slivers=[], options={}): filtre = {} - #tags_filter = {} - #if slice and 'node_ids' in slice and slice['node_ids']: - #filter['node_id'] = slice['node_ids'] - #tags_filter=filter.copy() - #filter.update({'peer_id': None}) nodes = self.driver.GetNodes(filtre) - - #site_ids = [] + + interface_ids = [] tag_ids = [] nodes_dict = {} + for node in nodes: + nodes_dict[node['node_id']] = node + + rspec_nodes = [] + for node in nodes: + + node['hostname'] = hostname_to_hrn( self.driver.root_auth,node['site_login_base'], node['hostname']) + rspec_node = Node() + + rspec_node['component_id'] = hostname_to_urn(self.driver.root_auth, node['site_login_base'], node['hostname']) + rspec_node['component_name'] = node['hostname'] + rspec_node['component_manager_id'] = hrn_to_urn(self.driver.root_auth, 'authority+sa') + rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.root_auth, node['site_login_base']), 'authority+sa') + rspec_node['boot_state'] = node['boot_state'] + if node['posx'] and node['posy']: + location = Location({'longitude':node['posx'], 'latitude': node['posy']}) + rspec_node['location'] = location + + rspec_node['exclusive'] = 'True' + rspec_node['hardware_types']= [HardwareType({'name': 'senslab sensor node'})] + + rspec_node['interfaces'] = [] + + + rspec_node['tags'] = [] + + rspec_nodes.append(rspec_node) + return (rspec_nodes) + + def get_nodes_and_links(self, slice=None,slivers=[], options={}): + filter = {} + tags_filter = {} + if slice and 'node_ids' in slice and slice['node_ids']: + filter['hostname'] = slice['node_ids'] + tags_filter=filter.copy() + + #geni_available = options.get('geni_available') + #if geni_available: + #filter['boot_state'] = 'boot' + print>>sys.stderr, "\r\n \r\n \t get_nodes_and_links filter %s \r\n \r\n \t slivers %s" %(filter, slivers) + #filter.update({'peer_id': None}) + nodes = self.driver.GetNodes(filter['hostname']) + print>>sys.stderr, "\r\n \r\n \t get_nodes_and_links nodes %s" %(nodes) + + #site_ids = [] + #interface_ids = [] + #tag_ids = [] + nodes_dict = {} for node in nodes: #site_ids.append(node['site_id']) #interface_ids.extend(node['interface_ids']) #tag_ids.extend(node['node_tag_ids']) nodes_dict[node['node_id']] = node - + # get sites #sites_dict = self.get_sites({'site_id': site_ids}) # get interfaces #interfaces = self.get_interfaces({'interface_id':interface_ids}) # get tags #node_tags = self.get_node_tags(tags_filter) - # get initscripts - #pl_initscripts = self.get_pl_initscripts() - + #links = self.get_links(sites_dict, nodes_dict, interfaces) - + rspec_nodes = [] for node in nodes: # skip whitelisted nodes #if node['slice_ids_whitelist']: #if not slice or slice['slice_id'] not in node['slice_ids_whitelist']: #continue - node['hostname'] = hostname_to_hrn( self.driver.root_auth,node['site_login_base'], node['hostname']) rspec_node = Node() # xxx how to retrieve site['login_base'] #site_id=node['site_id'] #site=sites_dict[site_id] - rspec_node['component_id'] = hostname_to_urn(self.driver.root_auth, node['site_login_base'], node['hostname']) - rspec_node['component_name'] = node['hostname'] + rspec_node['component_name'] = node['hostname'] rspec_node['component_manager_id'] = hrn_to_urn(self.driver.root_auth, 'authority+sa') + #rspec_node['component_manager_id'] = Xrn(self.driver.root_auth, 'authority+sa').get_urn() rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.root_auth, node['site_login_base']), 'authority+sa') - rspec_node['boot_state'] = node['boot_state'] - if node['posx'] and node['posy']: - location = Location({'longitude':node['posx'], 'latitude': node['posy']}) - rspec_node['location'] = location - - rspec_node['exclusive'] = 'True' - rspec_node['hardware_types']= [HardwareType({'name': 'senslab sensor node'})] + # do not include boot state ( element) in the manifest rspec + if not slice: + rspec_node['boot_state'] = node['boot_state'] + rspec_node['exclusive'] = 'true' + rspec_node['hardware_types'] = [HardwareType({'name': 'slab-sensor'})] # only doing this because protogeni rspec needs # to advertise available initscripts - #rspec_node['pl_initscripts'] = pl_initscripts.values() - # add site/interface info to nodes. + rspec_node['pl_initscripts'] = None + # add site/interface info to nodes. # assumes that sites, interfaces and tags have already been prepared. #site = sites_dict[node['site_id']] #if site['longitude'] and site['latitude']: - #location = Location({'longitude': site['longitude'], 'latitude': site['latitude']}) + #location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'}) #rspec_node['location'] = location + if node['posx'] and node['posy']: + location = Location({'longitude':node['posx'], 'latitude': node['posy']}) + rspec_node['location'] = location rspec_node['interfaces'] = [] #if_count=0 #for if_id in node['interface_ids']: #interface = Interface(interfaces[if_id]) #interface['ipv4'] = interface['ip'] - #interface['component_id'] = PlXrn(auth=self.api.hrn, interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn() + #interface['component_id'] = PlXrn(auth=self.driver.hrn, + #interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn() + # interfaces in the manifest need a client id + #if slice: + #interface['client_id'] = "%s:%s" % (node['node_id'], if_id) #rspec_node['interfaces'].append(interface) #if_count+=1 - + #tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']] rspec_node['tags'] = [] - #if node['node_id'] in slivers: - ## add sliver info - #sliver = slivers[node['node_id']] - #rspec_node['sliver_id'] = sliver['sliver_id'] - #rspec_node['client_id'] = node['hostname'] - #rspec_node['slivers'] = [sliver] + if node['hrn'] in slivers: + # add sliver info + sliver = slivers[node['node_id']] + rspec_node['sliver_id'] = sliver['sliver_id'] + rspec_node['client_id'] = node['hostname'] + rspec_node['slivers'] = [sliver] - ## slivers always provide the ssh service - #login = Login({'authentication': 'ssh-keys', 'hostname': node['hostname'], 'port':'22'}) + # slivers always provide the ssh service + #login = Login({'authentication': 'ssh-keys', 'hostname': node['hostname'], 'port':'22', 'username': sliver['name']}) #service = Services({'login': login}) #rspec_node['services'] = [service] rspec_nodes.append(rspec_node) - return (rspec_nodes) - - + return (rspec_nodes) #from plc/aggregate.py def get_rspec(self, slice_xrn=None, version = None, options={}): @@ -209,7 +252,7 @@ class SlabAggregate: #rspec.xml.set('expires', datetime_to_epoch(slice['expires'])) # add sliver defaults #nodes, links = self.get_nodes_and_links(slice, slivers) - nodes = self.get_nodes() + nodes = self.get_nodes_and_links(slice,slivers) rspec.version.add_nodes(nodes) #rspec.version.add_links(links) diff --git a/sfa/senslab/slabdriver.py b/sfa/senslab/slabdriver.py index fe9762c6..bf0bf90c 100644 --- a/sfa/senslab/slabdriver.py +++ b/sfa/senslab/slabdriver.py @@ -369,22 +369,38 @@ class SlabDriver(Driver): if job_id and resources is False: req = "GET_jobs_id" - node_list = 'assigned_network_address' + node_list_k = 'assigned_network_address' if job_id and resources : req = "GET_jobs_id_resources" - node_list = 'reserverd_resources' + node_list_k = 'reserverd_resources' #Get job info from OAR job_info = self.oar.parser.SendRequest(req, job_id) if job_info['state'] == 'Terminated': print>>sys.stderr, "\r\n \r\n \t\t GetJobs TERMINELEBOUSIN " return None - + + #Get a dict of nodes . Key :hostname of the node + node_list = self.GetNodes() + node_hostname_list = [] + for node in node_list: + node_hostname_list.append(node['hostname']) + node_dict = dict(zip(node_hostname_list,node_list)) + + print>>sys.stderr, "\r\n \r\n \r\n \r\n \r\n \t\t GetJobs GetNODES %s " %(node_list) try : - for n in job_info[node_list]: - n = str(self.root_auth) + str(n) - print>>sys.stderr, "\r\n \r\n \t\t GetJobs resources job_info %s" %(job_info) + #for n in job_info[node_list]: + #n = str(self.root_auth) + str(n) + + liste =job_info[node_list_k] + print>>sys.stderr, "\r\n \r\n \t\t GetJobs resources job_info liste%s" %(liste) + for k in range(len(liste)): + job_info[node_list_k][k] = node_dict[job_info[node_list_k][k]]['hostname'] + + print>>sys.stderr, "\r\n \r\n \t\t YYYYYYYYYYYYGetJobs resources job_info %s" %(job_info) + job_info.update({'node_ids':job_info[node_list_k]}) + del job_info[node_list_k] return job_info except KeyError: @@ -392,11 +408,7 @@ class SlabDriver(Driver): - - - - - + def GetNodes(self,node_filter= None, return_fields=None):