From: Sandrine Avakian Date: Tue, 24 Jan 2012 09:33:40 +0000 (+0100) Subject: Thrash commit between cleaning OARrestapi and adding job features... X-Git-Tag: sfa-2.1-24~3^2~231 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=0848f67382b2bdba46e0a0278bf1821f462e34d1;p=sfa.git Thrash commit between cleaning OARrestapi and adding job features... --- diff --git a/sfa/senslab/OARrestapi.py b/sfa/senslab/OARrestapi.py index 221b1198..427156b0 100644 --- a/sfa/senslab/OARrestapi.py +++ b/sfa/senslab/OARrestapi.py @@ -46,7 +46,8 @@ class OARrestapi: self.oarserver['ip'] = OARIP self.oarserver['port'] = 80 self.oarserver['uri'] = None - self.oarserver['postformat'] = 'json' + self.oarserver['postformat'] = 'json' + self.parser = OARGETParser(self) def GETRequestToOARRestAPI(self, request, strval=None ): self.oarserver['uri'] = OARrequests_get_uri_dict[request] @@ -223,8 +224,19 @@ class OARGETParser: self.raw_json = self.raw_json['items'] self.ParseNodes() self.ParseSites() + return self.node_dictlist - + resources_fulljson_dict= { + 'resource_id' : AddNodeId, + 'network_address' : AddNodeNetworkAddr, + 'site': AddNodeSite, + 'radio': AddNodeRadio, + 'mobile': AddMobility, + 'posx': AddPosX, + 'posy': AddPosY, + 'state':AddBootState, + } + #Parse nodes properties from OAR #Put them into a dictionary with key = node id and value is a dictionary @@ -234,13 +246,14 @@ class OARGETParser: #print >>sys.stderr, " \r\n \r\n \t\t OARrestapi.py ParseNodes self.raw_json %s" %(self.raw_json) for dictline in self.raw_json: #print >>sys.stderr, " \r\n \r\n \t\t OARrestapi.py ParseNodes dictline %s hey" %(dictline) - for k in dictline.keys(): + for k in dictline: if k in self.resources_fulljson_dict: # dictionary is empty and/or a new node has to be inserted if node_id is None : node_id = self.resources_fulljson_dict[k](self,self.node_dictlist, dictline[k]) else: ret = self.resources_fulljson_dict[k](self,self.node_dictlist[node_id], dictline[k]) + #If last property has been inserted in the property tuple list, reset node_id if ret == 0: #Turn the property tuple list (=dict value) into a dictionary @@ -304,16 +317,6 @@ class OARGETParser: 'GET_resources_full': {'uri':'/oarapi/resources/full.json','parse_func': ParseResourcesFull}, 'GET_resources':{'uri':'/oarapi/resources.json' ,'parse_func': ParseResources}, } - resources_fulljson_dict= { - 'resource_id' : AddNodeId, - 'network_address' : AddNodeNetworkAddr, - 'site': AddNodeSite, - 'radio': AddNodeRadio, - 'mobile': AddMobility, - 'posx': AddPosX, - 'posy': AddPosY, - 'state':AddBootState, - } def __init__(self, srv ): @@ -324,29 +327,31 @@ class OARGETParser: self.jobs_details_json_dict = self.jobs_json_dict self.server = srv self.node_dictlist = {} + self.site_dict = {} self.SendRequest("GET_version") def SendRequest(self,request, strval = None ): if request in OARrequests_get_uri_dict: - self.raw_json = self.server.GETRequestToOARRestAPI(request,strval) - self.OARrequests_uri_dict[request]['parse_func'](self) + self.raw_json = self.server.GETRequestToOARRestAPI(request,strval) + #print>>sys.stderr, "\r\n OARGetParse __init__ : request %s result %s "%(request,self.raw_json) + return self.OARrequests_uri_dict[request]['parse_func'](self) else: print>>sys.stderr, "\r\n OARGetParse __init__ : ERROR_REQUEST " ,request -class OARapi: +#class OARapi: + + #def __init__(self): + #self.server = OARrestapi() + #self.parser = OARGETParser(self.server) - def __init__(self): - self.server = OARrestapi() - self.parser = OARGETParser(self.server) - #GetNodes moved to slabdriver.py - def GetJobs(self): - print>>sys.stderr, " \r\n GetJobs" - self.parser.SendRequest("GET_jobs") - return self.parser.GetJobsFromOARParse() + #def GetJobs(self): + #print>>sys.stderr, " \r\n GetJobs" + #self.parser.SendRequest("GET_jobs") + #return self.parser.GetJobsFromOARParse() diff --git a/sfa/senslab/slab-import.py b/sfa/senslab/slab-import.py index 62690aae..37398a8f 100644 --- a/sfa/senslab/slab-import.py +++ b/sfa/senslab/slab-import.py @@ -2,7 +2,7 @@ import sys import datetime import time -from sfa.senslab.OARrestapi import OARapi +#from sfa.senslab.OARrestapi import OARapi from sfa.senslab.LDAPapi import LDAPapi from sfa.senslab.slabdriver import SlabDriver from sfa.senslab.slabpostgres import SlabDB diff --git a/sfa/senslab/slabaggregate.py b/sfa/senslab/slabaggregate.py index f913df79..f33eef4b 100644 --- a/sfa/senslab/slabaggregate.py +++ b/sfa/senslab/slabaggregate.py @@ -19,7 +19,7 @@ from sfa.rspecs.elements.location import Location from sfa.rspecs.elements.hardware_type import HardwareType from sfa.rspecs.elements.node import Node #from sfa.rspecs.elements.link import Link -#from sfa.rspecs.elements.sliver import Sliver +from sfa.rspecs.elements.sliver import Sliver #from sfa.rspecs.elements.login import Login #from sfa.rspecs.elements.location import Location #from sfa.rspecs.elements.interface import Interface @@ -48,7 +48,7 @@ class SlabAggregate: user_options = {} def __init__(self ,driver): - self.OARImporter = OARapi() + #self.OARImporter = OARapi() self.driver = driver #self.api = api print >>sys.stderr,"\r\n \r\n \t\t_____________INIT Slabaggregate api : %s" %(driver) @@ -65,19 +65,25 @@ class SlabAggregate: slice_urn = hrn_to_urn(slice_xrn, 'slice') slice_hrn, _ = urn_to_hrn(slice_xrn) slice_name = slice_hrn - slices = self.driver.GetSlices([slice_name]) + print >>sys.stderr,"\r\n \r\n \t\t_____________ Slabaggregate api get_slice_and_slivers " + slices = self.driver.GetSlices({'slice_hrn':str(slice_name)}) + print >>sys.stderr,"\r\n \r\n \t\t_____________ Slabaggregate api get_slice_and_slivers slices %s " %(slices) if not slices: return (slice, slivers) slice = slices[0] - ## sort slivers by node id - #for node_id in slice['node_ids']: - #sliver = Sliver({'sliver_id': urn_to_sliver_id(slice_urn, slice['slice_id'], node_id), - #'name': slice['hrn'], - #'type': 'slab-vm', - #'tags': []}) - #slivers[node_id]= sliver - + # sort slivers by node id + try: + + for node_id in slice['assigned_network_address']: + node_id = self.driver.root_auth + '.' + node_id + sliver = Sliver({'sliver_id': urn_to_sliver_id(slice_urn, slice['record_id_slice'], node_id), + 'name': slice['slice_hrn'], + 'type': 'slab-vm', + 'tags': []}) + slivers[node_id]= sliver + except KeyError: + print>>sys.stderr, " \r\n \t\t get_slice_and_slivers KeyError " ## sort sliver attributes by node id ##tags = self.driver.GetSliceTags({'slice_tag_id': slice['slice_tag_ids']}) ##for tag in tags: @@ -185,13 +191,11 @@ class SlabAggregate: #from plc/aggregate.py def get_rspec(self, slice_xrn=None, version = None, options={}): - print>>sys.stderr, " \r\n SlabAggregate \t\t get_rspec **************\r\n" - - + rspec = None version_manager = VersionManager() version = version_manager.get_version(version) - + print>>sys.stderr, " \r\n SlabAggregate \t\t get_rspec ************** version %s version_manager %s \r\n" %(version,version_manager) if not slice_xrn: rspec_version = version_manager._get_version(version.type, version.version, 'ad') @@ -207,11 +211,13 @@ class SlabAggregate: rspec.version.add_nodes(nodes) #rspec.version.add_links(links) - #default_sliver = slivers.get(None, []) - #if default_sliver: - #default_sliver_attribs = default_sliver.get('tags', []) - #for attrib in default_sliver_attribs: - #logger.info(attrib) - #rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value']) + default_sliver = slivers.get(None, []) + if default_sliver: + default_sliver_attribs = default_sliver.get('tags', []) + print>>sys.stderr, " \r\n SlabAggregate \t\t get_rspec ************** default_sliver_attribs %s \r\n" %(default_sliver_attribs) + for attrib in default_sliver_attribs: + print>>sys.stderr, " \r\n SlabAggregate \t\t get_rspec ************** attrib %s \r\n" %(attrib) + logger.info(attrib) + rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value']) return rspec.toxml() diff --git a/sfa/senslab/slabdriver.py b/sfa/senslab/slabdriver.py index 8f0bde79..44edfe4c 100644 --- a/sfa/senslab/slabdriver.py +++ b/sfa/senslab/slabdriver.py @@ -21,7 +21,7 @@ from sfa.util.plxrn import slicename_to_hrn, hostname_to_hrn, hrn_to_pl_slicenam # SlabDriver should be really only about talking to the senslab testbed ## thierry : please avoid wildcard imports :) -from sfa.senslab.OARrestapi import OARapi, OARrestapi +from sfa.senslab.OARrestapi import OARrestapi from sfa.senslab.LDAPapi import LDAPapi from sfa.senslab.SenslabImportUsers import SenslabImportUsers from sfa.senslab.parsing import parse_filter @@ -57,7 +57,8 @@ class SlabDriver(Driver): # thierry - just to not break the rest of this code - self.oar = OARapi() + #self.oar = OARapi() + self.oar = OARrestapi() self.ldap = LDAPapi() self.users = SenslabImportUsers() self.time_format = "%Y-%m-%d %H:%M:%S" @@ -187,6 +188,7 @@ class SlabDriver(Driver): #return slices # get data from db + print>>sys.stderr, " \r\n \t\t SLABDRIVER.PY list_slices" slices = self.GetSlices() slice_hrns = [slicename_to_hrn(self.hrn, slice['slice_hrn']) for slice in slices] slice_urns = [hrn_to_urn(slice_hrn, 'slice') for slice_hrn in slice_hrns] @@ -214,7 +216,8 @@ class SlabDriver(Driver): acceptable_fields=['url', 'instantiation', 'name', 'description'] for key in pl_record.keys(): if key not in acceptable_fields: - pl_record.pop(key) + pl_record.pop(key) + print>>sys.stderr, " \r\n \t\t SLABDRIVER.PY register" slices = self.GetSlices([pl_record['hrn']]) if not slices: pointer = self.AddSlice(pl_record) @@ -355,14 +358,23 @@ class SlabDriver(Driver): if return_person_list: print>>sys.stderr, " \r\n GetPersons person_filter %s return_fields %s return_person_list %s " %(person_filter,return_fields,return_person_list) return return_person_list - - + + def GetJobs(self,job_id= None, resources=True,return_fields=None, details = None): + job_resources=['reserved_resources', 'assigned_resources','job_id', 'job_uri', 'assigned_nodes',\ + 'api_timestamp'] + assigned_res = ['resource_id', 'resource_uri'] + assigned_n = ['node', 'node_uri'] + + if job_id and resources is False: - self.oar.parser.SendRequest("GET_jobs_id", job_id) - if job_id and resources: - self.oar.parser.SendRequest("GET_jobs_id_resources", job_id) - + job_info = self.oar.parser.SendRequest("GET_jobs_id", job_id) + print>>sys.stderr, "\r\n \r\n \t\t GetJobs resources is False job_info %s" %(job_info) + + if job_id and resources : + job_info = self.oar.parser.SendRequest("GET_jobs_id_resources", job_id) + print>>sys.stderr, "\r\n \r\n \t\t GetJobs job_info %s" %(job_info) + return job_info #node_dict = self.oar.parser.GetNodesFromOARParse() #return_node_list = [] @@ -375,10 +387,10 @@ class SlabDriver(Driver): def GetNodes(self,node_filter= None, return_fields=None): - self.oar.parser.SendRequest("GET_resources_full") - node_dict = self.oar.parser.GetNodesFromOARParse() + node_dict =self.oar.parser.SendRequest("GET_resources_full") + #node_dict = self.oar.parser.GetNodesFromOARParse() return_node_list = [] - + print>>sys.stderr, "\r\n \r\n \t\t GetNodes node_dict %s" %(node_dict) if not (node_filter or return_fields): return_node_list = node_dict.values() return return_node_list @@ -401,7 +413,7 @@ class SlabDriver(Driver): def GetSlices(self,slice_filter = None, return_fields=None): return_slice_list =[] - sliceslist = self.db.find('slice',columns = ['oar_job_id', 'slice_hrn', 'record_id_slice','record_id_user']) + sliceslist = self.db.find('slice',columns = ['oar_job_id', 'slice_hrn', 'record_id_slice','record_id_user'], record_filter=slice_filter) print >>sys.stderr, " \r\n \r\n SLABDRIVER.PY GetSlices slices %s slice_filter %s " %(sliceslist,slice_filter) #slicesdict = sliceslist[0] @@ -410,14 +422,20 @@ class SlabDriver(Driver): for sl in sliceslist: if sl['oar_job_id'] is not -1: print >>sys.stderr, " \r\n \r\n SLABDRIVER.PY GetSlices sl %s" %(sl) - self.GetJobs( sl['oar_job_id'],resources=False) + rslt = self.GetJobs( sl['oar_job_id'],resources=False) + print >>sys.stderr, " \r\n \r\n SLABRIVER.PY GetSlices rslt %s" %(rslt) + sl.update(rslt) return_slice_list = sliceslist return return_slice_list return_slice_list = parse_filter(sliceslist, slice_filter,'slice', return_fields) for sl in return_slice_list: - if sl['oar_job_id'] is not -1: - self.GetJobs( sl['oar_job_id'],resources=False) + if sl['oar_job_id'] is not -1: + print >>sys.stderr, " \r\n \r\n SLABDRIVER.PY GetSlices sl %s" %(sl) + rslt =self.GetJobs( sl['oar_job_id'],resources=False) + print >>sys.stderr, " \r\n \r\n SLABRIVER.PY GetSlices rslt %s" %(rslt) + sl.update(rslt) + #print >>sys.stderr, " \r\n \r\n SLABDRIVER.PY GetSlices return_slice_list %s" %(return_slice_list) return return_slice_list @@ -527,7 +545,7 @@ class SlabDriver(Driver): reqdict['resource'] ="network_address="+ str(len(nodeid_list)) reqdict['resource']+= ",walltime=" + str(00) + ":" + str(05) + ":" + str(00) reqdict['script_path'] = "/bin/sleep 320" - reqdict['type'] = "deploy" + #reqdict['type'] = "deploy" print>>sys.stderr, "\r\n \r\n AddSliceToNodes reqdict %s \r\n site_list %s" %(reqdict,site_list) OAR = OARrestapi() answer = OAR.POSTRequestToOARRestAPI('POST_job',reqdict,slice_user) @@ -667,29 +685,41 @@ class SlabDriver(Driver): fields in the record. """ print >>sys.stderr, "\r\n \t\t BEFORE fill_record_info %s" %(records) - if isinstance(records, list): + if not isinstance(records, list): records = [records] #print >>sys.stderr, "\r\n \t\t BEFORE fill_record_pl_info %s" %(records) - + parkour = records try: - if records['type'] == 'slice': - - sfatable = SfaTable() - recslice = self.db.find('slice',str(records['hrn'])) - if isinstance(recslice,list) and len(recslice) == 1: - recslice = recslice[0] - recuser = sfatable.find( recslice['record_id_user'], ['hrn']) - - print >>sys.stderr, "\r\n \t\t SLABDRIVER.PY fill_record_info %s" %(recuser) - records['type'] - if isinstance(recuser,list) and len(recuser) == 1: - recuser = recuser[0] - records.update({'PI':[recuser['hrn']], - 'researcher': [recuser['hrn']], - 'name':records['hrn'], 'oar_job_id':recslice['oar_job_id'], - - 'node_ids': [], - 'person_ids':[recslice['record_id_user']]}) + for record in parkour: + + if str(record['type']) == 'slice': + print >>sys.stderr, "\r\n \t\t SLABDRIVER.PY fill_record_info record %s" %(record) + sfatable = SfaTable() + recslice = self.db.find('slice',str(record['hrn'])) + if isinstance(recslice,list) and len(recslice) == 1: + recslice = recslice[0] + recuser = sfatable.find( recslice['record_id_user'], ['hrn']) + + print >>sys.stderr, "\r\n \t\t SLABDRIVER.PY fill_record_info %s" %(recuser) + + if isinstance(recuser,list) and len(recuser) == 1: + recuser = recuser[0] + record.update({'PI':[recuser['hrn']], + 'researcher': [recuser['hrn']], + 'name':record['hrn'], + 'oar_job_id':recslice['oar_job_id'], + 'node_ids': [], + 'person_ids':[recslice['record_id_user']]}) + + elif str(record['type']) == 'user': + recslice = self.db.find('slice', record_filter={'record_id_user':record['record_id']}) + for rec in recslice: + rec.update({'type':'slice'}) + rec.update({'hrn':rec['slice_hrn'], 'record_id':rec['record_id_slice']}) + records.append(rec) + print >>sys.stderr, "\r\n \t\t SLABDRIVER.PY fill_record_info ADDING SLIC EINFO recslice %s" %(recslice) + + except TypeError: print >>sys.stderr, "\r\n \t\t SLABDRIVER fill_record_info EXCEPTION RECORDS : %s" %(records) return diff --git a/sfa/senslab/slabpostgres.py b/sfa/senslab/slabpostgres.py index aaaf5ca8..668c8c39 100644 --- a/sfa/senslab/slabpostgres.py +++ b/sfa/senslab/slabpostgres.py @@ -241,7 +241,8 @@ class SlabDB: return - def find(self, tablename,record_filter = None, columns=None): + def find(self, tablename,record_filter = None, columns=None): + print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES.PY find : record_filter %s %s columns %s %s" %( record_filter , type(record_filter),columns , type(columns)) if not columns: columns = "*" else: @@ -265,8 +266,9 @@ class SlabDB: if isinstance(record_filter, dict): for k in record_filter.keys(): - sql += "AND "+' \''+ str(k) + '\''+ '='+' \''+ str(record_filter[k])+'\'' - + #sql += "AND "+' \''+ str(k) + '\''+ '='+' \''+ str(record_filter[k])+'\'' + #sql += "AND "+ str(k) + '=' + str(record_filter[k]) + sql += "AND "+ str(k) +'='+' \''+ str(record_filter[k])+'\'' elif isinstance(record_filter, str): sql += "AND slice_hrn ="+ ' \''+record_filter+'\''