X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=sfa%2Fiotlab%2FOARrestapi.py;h=186790806091adaafe64841f3362ebe59c04fc8c;hb=a18ad5b1dbc2a1dd346784105c89a0714086ebc4;hp=0223f472524fa1e6cd598423a7f2948c33a00cc8;hpb=efa9ed860628dba98c63b7722f472d922fbe8a87;p=sfa.git diff --git a/sfa/iotlab/OARrestapi.py b/sfa/iotlab/OARrestapi.py index 0223f472..18679080 100644 --- a/sfa/iotlab/OARrestapi.py +++ b/sfa/iotlab/OARrestapi.py @@ -1,32 +1,35 @@ -#import sys +""" +File used to handle issuing request to OAR and parse OAR's JSON responses. +Contains the following classes: +- JsonPage : handles multiple pages OAR answers. +- OARRestapi : handles issuing POST or GET requests to OAR. +- ParsingResourcesFull : dedicated to parsing OAR's answer to a get resources +full request. +- OARGETParser : handles parsing the Json answers to different GET requests. + +""" from httplib import HTTPConnection, HTTPException, NotConnected import json -#import datetime -#from time import gmtime, strftime -import os.path -import sys -#import urllib -#import urllib2 from sfa.util.config import Config -#from sfa.util.xrn import hrn_to_urn, get_authority, Xrn, get_leaf - from sfa.util.sfalogging import logger +import os.path -OAR_REQUEST_POST_URI_DICT = {'POST_job':{'uri': '/oarapi/jobs.json'}, - 'DELETE_jobs_id':{'uri':'/oarapi/jobs/id.json'}, - } - -POST_FORMAT = {'json' : {'content':"application/json", 'object':json},} +class JsonPage: -#OARpostdatareqfields = {'resource' :"/nodes=", 'command':"sleep", \ - #'workdir':"/home/", 'walltime':""} + """Class used to manipulate json pages given by OAR. + In case the json answer from a GET request is too big to fit in one json + page, this class provides helper methods to retrieve all the pages and + store them in a list before putting them into one single json dictionary, + facilitating the parsing. + """ -class JsonPage: - """Class used to manipulate jsopn pages given by OAR.""" def __init__(self): + """Defines attributes to manipulate and parse the json pages. + + """ #All are boolean variables self.concatenate = False #Indicates end of data, no more pages to be loaded. @@ -38,11 +41,12 @@ class JsonPage: self.raw_json = None def FindNextPage(self): - """ Gets next data page from OAR when the query's results - are too big to be transmitted in a single page. - Uses the "links' item in the json returned to check if - an additionnal page has to be loaded. - Returns : next page , next offset query + """ + Gets next data page from OAR when the query's results are too big to + be transmitted in a single page. Uses the "links' item in the json + returned to check if an additionnal page has to be loaded. Updates + object attributes next_page, next_offset, and end. + """ if "links" in self.raw_json: for page in self.raw_json['links']: @@ -50,10 +54,9 @@ class JsonPage: self.concatenate = True self.next_page = True self.next_offset = "?" + page['href'].split("?")[1] - print>>sys.stderr, "\r\n \t FindNextPage NEXT LINK" return - if self.concatenate : + if self.concatenate: self.end = True self.next_page = False self.next_offset = None @@ -70,6 +73,24 @@ class JsonPage: @staticmethod def ConcatenateJsonPages(saved_json_list): + """ + If the json answer is too big to be contained in a single page, + all the pages have to be loaded and saved before being appended to the + first page. + + :param saved_json_list: list of all the stored pages, including the + first page. + :type saved_json_list: list + :returns: Returns a dictionary with all the pages saved in the + saved_json_list. The key of the dictionary is 'items'. + :rtype: dict + + + .. seealso:: SendRequest + .. warning:: Assumes the apilib is 0.2.10 (with the 'items' key in the + raw json dictionary) + + """ #reset items list tmp = {} @@ -79,8 +100,13 @@ class JsonPage: tmp['items'].extend(page['items']) return tmp - def ResetNextPage(self): + """ + Resets all the Json page attributes (next_page, next_offset, + concatenate, end). Has to be done before getting another json answer + so that the previous page status does not affect the new json load. + + """ self.next_page = True self.next_offset = None self.concatenate = False @@ -88,10 +114,25 @@ class JsonPage: class OARrestapi: - def __init__(self, config_file = '/etc/sfa/oar_config.py'): - self.oarserver = {} + """Class used to connect to the OAR server and to send GET and POST + requests. + + """ + + # classes attributes + + OAR_REQUEST_POST_URI_DICT = {'POST_job': {'uri': '/oarapi/jobs.json'}, + 'DELETE_jobs_id': + {'uri': '/oarapi/jobs/id.json'}, + } + + POST_FORMAT = {'json': {'content': "application/json", 'object': json}} + #OARpostdatareqfields = {'resource' :"/nodes=", 'command':"sleep", \ + #'workdir':"/home/", 'walltime':""} + def __init__(self, config_file='/etc/sfa/oar_config.py'): + self.oarserver = {} self.oarserver['uri'] = None self.oarserver['postformat'] = 'json' @@ -104,30 +145,57 @@ class OARrestapi: except IOError: raise IOError, "Could not find or load the configuration file: %s" \ - % config_file + % config_file #logger.setLevelDebug() self.oarserver['ip'] = self.OAR_IP self.oarserver['port'] = self.OAR_PORT - self.jobstates = ['Terminated', 'Hold', 'Waiting', 'toLaunch', \ - 'toError', 'toAckReservation', 'Launching', \ - 'Finishing', 'Running', 'Suspended', 'Resuming',\ - 'Error'] + self.jobstates = ['Terminated', 'Hold', 'Waiting', 'toLaunch', + 'toError', 'toAckReservation', 'Launching', + 'Finishing', 'Running', 'Suspended', 'Resuming', + 'Error'] self.parser = OARGETParser(self) - def GETRequestToOARRestAPI(self, request, strval=None, next_page=None, username = None ): + def GETRequestToOARRestAPI(self, request, strval=None, + next_page=None, username=None): + + """Makes a GET request to OAR. + + Fetch the uri associated with the resquest stored in + OARrequests_uri_dict, adds the username if needed and if available, adds + strval to the request uri if needed, connects to OAR and issues the GET + request. Gets the json reply. + + :param request: One of the known get requests that are keys in the + OARrequests_uri_dict. + :param strval: used when a job id has to be specified. + :param next_page: used to tell OAR to send the next page for this + Get request. Is appended to the GET uri. + :param username: used when a username has to be specified, when looking + for jobs scheduled by a particular user for instance. + + :type request: string + :type strval: integer + :type next_page: boolean + :type username: string + :returns: a json dictionary if OAR successfully processed the GET + request. + + .. seealso:: OARrequests_uri_dict + """ self.oarserver['uri'] = \ - OARGETParser.OARrequests_uri_dict[request]['uri'] + OARGETParser.OARrequests_uri_dict[request]['uri'] #Get job details with username if 'owner' in OARGETParser.OARrequests_uri_dict[request] and username: - self.oarserver['uri'] += OARGETParser.OARrequests_uri_dict[request]['owner'] + username + self.oarserver['uri'] += \ + OARGETParser.OARrequests_uri_dict[request]['owner'] + username headers = {} data = json.dumps({}) - logger.debug("OARrestapi \tGETRequestToOARRestAPI %s" %(request)) + logger.debug("OARrestapi \tGETRequestToOARRestAPI %s" % (request)) if strval: self.oarserver['uri'] = self.oarserver['uri'].\ - replace("id",str(strval)) + replace("id", str(strval)) if next_page: self.oarserver['uri'] += next_page @@ -136,21 +204,21 @@ class OARrestapi: headers['X-REMOTE_IDENT'] = username logger.debug("OARrestapi: \t GETRequestToOARRestAPI \ - self.oarserver['uri'] %s strval %s" \ - %(self.oarserver['uri'], strval)) - try : + self.oarserver['uri'] %s strval %s" + % (self.oarserver['uri'], strval)) + try: #seems that it does not work if we don't add this headers['content-length'] = '0' - conn = HTTPConnection(self.oarserver['ip'], \ - self.oarserver['port']) + conn = HTTPConnection(self.oarserver['ip'], + self.oarserver['port']) conn.request("GET", self.oarserver['uri'], data, headers) - resp = ( conn.getresponse()).read() + resp = (conn.getresponse()).read() conn.close() - except HTTPException, error : - logger.log_exc("GET_OAR_SRVR : Problem with OAR server : %s " \ - %(error)) + except HTTPException, error: + logger.log_exc("GET_OAR_SRVR : Problem with OAR server : %s " + % (error)) #raise ServerError("GET_OAR_SRVR : Could not reach OARserver") try: js_dict = json.loads(resp) @@ -158,8 +226,8 @@ class OARrestapi: return js_dict except ValueError, error: - logger.log_exc("Failed to parse Server Response: %s ERROR %s"\ - %(js_dict, error)) + logger.log_exc("Failed to parse Server Response: %s ERROR %s" + % (js_dict, error)) #raise ServerError("Failed to parse Server Response:" + js) @@ -171,7 +239,8 @@ class OARrestapi: #first check that all params for are OK try: - self.oarserver['uri'] = OAR_REQUEST_POST_URI_DICT[request]['uri'] + self.oarserver['uri'] = \ + self.OAR_REQUEST_POST_URI_DICT[request]['uri'] except KeyError: logger.log_exc("OARrestapi \tPOSTRequestToOARRestAPI request not \ @@ -184,7 +253,7 @@ class OARrestapi: data = json.dumps(datadict) headers = {'X-REMOTE_IDENT':username, \ - 'content-type': POST_FORMAT['json']['content'], \ + 'content-type': self.POST_FORMAT['json']['content'], \ 'content-length':str(len(data))} try : @@ -202,7 +271,7 @@ class OARrestapi: try: answer = json.loads(resp) - logger.debug("POSTRequestToOARRestAPI : answer %s" %(answer)) + logger.debug("POSTRequestToOARRestAPI : answer %s" % (answer)) return answer except ValueError, error: @@ -211,79 +280,229 @@ class OARrestapi: #raise ServerError("Failed to parse Server Response:" + answer) +class ParsingResourcesFull(): + """ + Class dedicated to parse the json response from a GET_resources_full from + OAR. + + """ + def __init__(self): + """ + Set the parsing dictionary. Works like a switch case, if the key is + found in the dictionary, then the associated function is called. + This is used in ParseNodes to create an usable dictionary from + the Json returned by OAR when issuing a GET resources full request. + + .. seealso:: ParseNodes + + """ + self.resources_fulljson_dict = { + 'network_address': self.AddNodeNetworkAddr, + 'site': self.AddNodeSite, + # 'radio': self.AddNodeRadio, + 'mobile': self.AddMobility, + 'x': self.AddPosX, + 'y': self.AddPosY, + 'z': self.AddPosZ, + 'archi': self.AddHardwareType, + 'state': self.AddBootState, + 'id': self.AddOarNodeId, + } + + + + def AddOarNodeId(self, tuplelist, value): + """Adds Oar internal node id to the nodes' attributes. + + Appends tuple ('oar_id', node_id) to the tuplelist. Used by ParseNodes. + + .. seealso:: ParseNodes + + """ + + tuplelist.append(('oar_id', int(value))) + + + def AddNodeNetworkAddr(self, dictnode, value): + """First parsing function to be called to parse the json returned by OAR + answering a GET_resources (/oarapi/resources.json) request. + + When a new node is found in the json, this function is responsible for + creating a new entry in the dictionary for storing information on this + specific node. The key is the node network address, which is also the + node's hostname. + The value associated with the key is a tuple list.It contains all + the nodes attributes. The tuplelist will later be turned into a dict. + + :param dictnode: should be set to the OARGETParser atribute + node_dictlist. It will store the information on the nodes. + :param value: the node_id is the network_address in the raw json. + :type value: string + :type dictnode: dictionary + + .. seealso: ParseResources, ParseNodes + """ + + node_id = value + dictnode[node_id] = [('node_id', node_id),('hostname', node_id) ] + + return node_id + + def AddNodeSite(self, tuplelist, value): + """Add the site's node to the dictionary. + + + :param tuplelist: tuple list on which to add the node's site. + Contains the other node attributes as well. + :param value: value to add to the tuple list, in this case the node's + site. + :type tuplelist: list + :type value: string + + .. seealso:: AddNodeNetworkAddr + + """ + tuplelist.append(('site', str(value))) + + # def AddNodeRadio(tuplelist, value): + # """Add thenode's radio chipset type to the tuple list. + + # :param tuplelist: tuple list on which to add the node's mobility + # status. The tuplelist is the value associated with the node's + # id in the OARGETParser + # 's dictionary node_dictlist. + # :param value: name of the radio chipset on the node. + # :type tuplelist: list + # :type value: string + + # .. seealso:: AddNodeNetworkAddr + + # """ + # tuplelist.append(('radio', str(value))) + + + def AddMobility(self, tuplelist, value): + """Add if the node is a mobile node or not to the tuple list. + + :param tuplelist: tuple list on which to add the node's mobility status. + The tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. + :param value: tells if a node is a mobile node or not. The value is found + in the json. + + :type tuplelist: list + :type value: integer + + .. seealso:: AddNodeNetworkAddr + + """ + if value is 0: + tuplelist.append(('mobile', 'False')) + else: + tuplelist.append(('mobile', 'True')) + + + def AddPosX(self, tuplelist, value): + """Add the node's position on the x axis. + + :param tuplelist: tuple list on which to add the node's position . The + tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. + :param value: the position x. + + :type tuplelist: list + :type value: integer + + .. seealso:: AddNodeNetworkAddr + + """ + tuplelist.append(('posx', value )) + + + + def AddPosY(self, tuplelist, value): + """Add the node's position on the y axis. + + :param tuplelist: tuple list on which to add the node's position . The + tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. + :param value: the position y. -def AddOarNodeId(tuplelist, value): - """ Adds Oar internal node id to the nodes attributes """ + :type tuplelist: list + :type value: integer - tuplelist.append(('oar_id', int(value))) + .. seealso:: AddNodeNetworkAddr + """ + tuplelist.append(('posy', value)) + + + + def AddPosZ(self, tuplelist, value): + """Add the node's position on the z axis. + + :param tuplelist: tuple list on which to add the node's position . The + tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. + :param value: the position z. -def AddNodeNetworkAddr(dictnode, value): - #Inserts new key. The value associated is a tuple list - node_id = value + :type tuplelist: list + :type value: integer - dictnode[node_id] = [('node_id', node_id),('hostname', node_id) ] + .. seealso:: AddNodeNetworkAddr - return node_id + """ + + tuplelist.append(('posz', value)) -def AddNodeSite(tuplelist, value): - tuplelist.append(('site', str(value))) -def AddNodeRadio(tuplelist, value): - tuplelist.append(('radio', str(value))) + def AddBootState(tself, tuplelist, value): + """Add the node's state, Alive or Suspected. -def AddMobility(tuplelist, value): - if value is 0: - tuplelist.append(('mobile', 'False')) - else : - tuplelist.append(('mobile', 'True')) + :param tuplelist: tuple list on which to add the node's state . The + tuplelist is the value associated with the node's id in the + OARGETParser 's dictionary node_dictlist. + :param value: node's state. -def AddPosX(tuplelist, value): - tuplelist.append(('posx', value)) + :type tuplelist: list + :type value: string -def AddPosY(tuplelist, value): - tuplelist.append(('posy', value)) + .. seealso:: AddNodeNetworkAddr -def AddPosZ(tuplelist, value): - tuplelist.append(('posz', value)) + """ + tuplelist.append(('boot_state', str(value))) -def AddBootState(tuplelist, value): - tuplelist.append(('boot_state', str(value))) -#Insert a new node into the dictnode dictionary -def AddNodeId(dictnode, value): - #Inserts new key. The value associated is a tuple list - node_id = int(value) + def AddHardwareType(self, tuplelist, value): + """Add the node's hardware model and radio chipset type to the tuple + list. - dictnode[node_id] = [('node_id', node_id)] - return node_id + :param tuplelist: tuple list on which to add the node's architecture + and radio chipset type. + :param value: hardware type: radio chipset. The value contains both the + architecture and the radio chipset, separated by a colon. + :type tuplelist: list + :type value: string -def AddHardwareType(tuplelist, value): - value_list = value.split(':') - tuplelist.append(('archi', value_list[0])) - tuplelist.append(('radio', value_list[1])) + .. seealso:: AddNodeNetworkAddr + + """ + + value_list = value.split(':') + tuplelist.append(('archi', value_list[0])) + tuplelist.append(('radio', value_list[1])) class OARGETParser: - resources_fulljson_dict = { - 'network_address' : AddNodeNetworkAddr, - 'site': AddNodeSite, - 'radio': AddNodeRadio, - 'mobile': AddMobility, - 'x': AddPosX, - 'y': AddPosY, - 'z':AddPosZ, - 'archi':AddHardwareType, - 'state':AddBootState, - 'id' : AddOarNodeId, - } + """Class providing parsing methods associated to specific GET requests. + """ - def __init__(self, srv) : + def __init__(self, srv): self.version_json_dict = { - 'api_version' : None , 'apilib_version' :None,\ - 'api_timezone': None, 'api_timestamp': None, 'oar_version': None ,} + 'api_version': None, 'apilib_version': None, + 'api_timezone': None, 'api_timestamp': None, 'oar_version': None} self.config = Config() self.interface_hrn = self.config.SFA_INTERFACE_HRN self.timezone_json_dict = { @@ -297,70 +516,116 @@ class OARGETParser: self.node_dictlist = {} self.json_page = JsonPage() - + self.parsing_resourcesfull = ParsingResourcesFull() self.site_dict = {} + self.jobs_list = [] self.SendRequest("GET_version") + def ParseVersion(self): + """Parses the OAR answer to the GET_version ( /oarapi/version.json.) + Finds the OAR apilib version currently used. Has an impact on the json + structure returned by OAR, so the version has to be known before trying + to parse the jsons returned after a get request has been issued. + Updates the attribute version_json_dict. + """ - def ParseVersion(self) : - #print self.json_page.raw_json - #print >>sys.stderr, self.json_page.raw_json - if 'oar_version' in self.json_page.raw_json : - self.version_json_dict.update(api_version = \ - self.json_page.raw_json['api_version'], - apilib_version = self.json_page.raw_json['apilib_version'], - api_timezone = self.json_page.raw_json['api_timezone'], - api_timestamp = self.json_page.raw_json['api_timestamp'], - oar_version = self.json_page.raw_json['oar_version'] ) - else : - self.version_json_dict.update(api_version = \ - self.json_page.raw_json['api'] , - apilib_version = self.json_page.raw_json['apilib'], - api_timezone = self.json_page.raw_json['api_timezone'], - api_timestamp = self.json_page.raw_json['api_timestamp'], - oar_version = self.json_page.raw_json['oar'] ) + if 'oar_version' in self.json_page.raw_json: + self.version_json_dict.update( + api_version=self.json_page.raw_json['api_version'], + apilib_version=self.json_page.raw_json['apilib_version'], + api_timezone=self.json_page.raw_json['api_timezone'], + api_timestamp=self.json_page.raw_json['api_timestamp'], + oar_version=self.json_page.raw_json['oar_version']) + else: + self.version_json_dict.update( + api_version=self.json_page.raw_json['api'], + apilib_version=self.json_page.raw_json['apilib'], + api_timezone=self.json_page.raw_json['api_timezone'], + api_timestamp=self.json_page.raw_json['api_timestamp'], + oar_version=self.json_page.raw_json['oar']) print self.version_json_dict['apilib_version'] - def ParseTimezone(self) : + def ParseTimezone(self): + """Get the timezone used by OAR. + + Get the timezone from the answer to the GET_timezone request. + :return: api_timestamp and api timezone. + :rype: integer, integer + + .. warning:: unused. + """ api_timestamp = self.json_page.raw_json['api_timestamp'] api_tz = self.json_page.raw_json['timezone'] return api_timestamp, api_tz - def ParseJobs(self) : + def ParseJobs(self): + """Called when a GET_jobs request has been issued to OAR. + + Corresponds to /oarapi/jobs.json uri. Currently returns the raw json + information dict. + :returns: json_page.raw_json + :rtype: dictionary + + .. warning:: Does not actually parse the information in the json. SA + 15/07/13. + + """ self.jobs_list = [] print " ParseJobs " return self.json_page.raw_json - def ParseJobsTable(self) : + def ParseJobsTable(self): + """In case we need to use the job table in the future. + + Associated with the GET_jobs_table : '/oarapi/jobs/table.json uri. + .. warning:: NOT USED. DOES NOTHING. + """ print "ParseJobsTable" - def ParseJobsDetails (self): - # currently, this function is not used a lot, - #so i have no idea what be usefull to parse, - #returning the full json. NT + def ParseJobsDetails(self): + """Currently only returns the same json in self.json_page.raw_json. + + .. todo:: actually parse the json + .. warning:: currently, this function is not used a lot, so I have no + idea what could be useful to parse, returning the full json. NT + """ + #logger.debug("ParseJobsDetails %s " %(self.json_page.raw_json)) return self.json_page.raw_json def ParseJobsIds(self): + """Associated with the GET_jobs_id OAR request. + + Parses the json dict (OAR answer) to the GET_jobs_id request + /oarapi/jobs/id.json. - job_resources = ['wanted_resources', 'name', 'id', 'start_time', \ - 'state','owner','walltime','message'] + :returns: dictionary whose keys are listed in the local variable + job_resources and values that are in the json dictionary returned + by OAR with the job information. + :rtype: dict - job_resources_full = ['launching_directory', 'links', \ - 'resubmit_job_id', 'owner', 'events', 'message', \ - 'scheduled_start', 'id', 'array_id', 'exit_code', \ - 'properties', 'state','array_index', 'walltime', \ - 'type', 'initial_request', 'stop_time', 'project',\ - 'start_time', 'dependencies','api_timestamp','submission_time', \ - 'reservation', 'stdout_file', 'types', 'cpuset_name', \ - 'name', 'wanted_resources','queue','stderr_file','command'] + """ + job_resources = ['wanted_resources', 'name', 'id', 'start_time', + 'state', 'owner', 'walltime', 'message'] + + # Unused variable providing the contents of the json dict returned from + # get job resources full request + job_resources_full = [ + 'launching_directory', 'links', + 'resubmit_job_id', 'owner', 'events', 'message', + 'scheduled_start', 'id', 'array_id', 'exit_code', + 'properties', 'state', 'array_index', 'walltime', + 'type', 'initial_request', 'stop_time', 'project', + 'start_time', 'dependencies', 'api_timestamp', 'submission_time', + 'reservation', 'stdout_file', 'types', 'cpuset_name', + 'name', 'wanted_resources', 'queue', 'stderr_file', 'command'] job_info = self.json_page.raw_json @@ -386,10 +651,9 @@ class OARGETParser: for resource in self.json_page.raw_json['items']: job_resources.append(resource['id']) - #logger.debug("OARESTAPI \tParseJobsIdResources %s" %(self.json_page.raw_json)) return job_resources - def ParseResources(self) : + def ParseResources(self): """ Parses the json produced by a get_resources request on oar.""" #logger.debug("OARESTAPI \tParseResources " ) @@ -398,31 +662,38 @@ class OARGETParser: self.ParseNodes() def ParseReservedNodes(self): - """ Returns an array containing the list of the reserved nodes """ + """ Returns an array containing the list of the jobs scheduled + with the reserved nodes if available. + + :returns: list of job dicts, each dict containing the following keys: + t_from, t_until, resources_ids (of the reserved nodes for this job). + If the information is not available, default values will be set for + these keys. The other keys are : state, lease_id and user. + :rtype: list + + """ #resources are listed inside the 'items' list from the json reservation_list = [] job = {} #Parse resources info - for json_element in self.json_page.raw_json['items']: + for json_element in self.json_page.raw_json['items']: #In case it is a real reservation (not asap case) if json_element['scheduled_start']: job['t_from'] = json_element['scheduled_start'] job['t_until'] = int(json_element['scheduled_start']) + \ - int(json_element['walltime']) + int(json_element['walltime']) #Get resources id list for the job - job['resource_ids'] = \ - [ node_dict['id'] for node_dict in json_element['resources']] + job['resource_ids'] = [node_dict['id'] for node_dict + in json_element['resources']] else: job['t_from'] = "As soon as possible" job['t_until'] = "As soon as possible" job['resource_ids'] = ["Undefined"] - job['state'] = json_element['state'] job['lease_id'] = json_element['id'] - job['user'] = json_element['owner'] #logger.debug("OARRestapi \tParseReservedNodes job %s" %(job)) reservation_list.append(job) @@ -434,16 +705,34 @@ class OARGETParser: """ Gets the list of nodes currently in use from the attributes of the running jobs. + :returns: list of hostnames, the nodes that are currently involved in + running jobs. + :rtype: list + + """ - logger.debug("OARESTAPI \tParseRunningJobs__________________________ ") + logger.debug("OARESTAPI \tParseRunningJobs_________________ ") #resources are listed inside the 'items' list from the json nodes = [] - for job in self.json_page.raw_json['items']: + for job in self.json_page.raw_json['items']: for node in job['nodes']: nodes.append(node['network_address']) return nodes + def ChangeRawJsonDependingOnApilibVersion(self): + """ + Check if the OAR apilib version is different from 0.2.10, in which case + the Json answer is also dict instead as a plain list. + + .. warning:: the whole code is assuming the json contains a 'items' key + .. seealso:: ConcatenateJsonPages, ParseJobs, ParseReservedNodes, + ParseJobsIdResources, ParseResources, ParseRunningJobs + .. todo:: Clean the whole code. Either suppose the apilib will always + provide the 'items' key, or handle different options. + """ + if self.version_json_dict['apilib_version'] != "0.2.10": + self.json_page.raw_json = self.json_page.raw_json['items'] def ParseDeleteJobs(self): """ No need to parse anything in this function.A POST @@ -452,36 +741,44 @@ class OARGETParser: """ return - def ParseResourcesFull(self) : + def ParseResourcesFull(self): """ This method is responsible for parsing all the attributes of all the nodes returned by OAR when issuing a get resources full. The information from the nodes and the sites are separated. Updates the node_dictlist so that the dictionnary of the platform's nodes is available afterwards. + :returns: node_dictlist, a list of dictionaries about the nodes and + their properties. + :rtype: list + """ - logger.debug("OARRESTAPI ParseResourcesFull________________________ ") + logger.debug("OARRESTAPI ParseResourcesFull___________ ") #print self.json_page.raw_json[1] #resources are listed inside the 'items' list from the json - if self.version_json_dict['apilib_version'] != "0.2.10" : - self.json_page.raw_json = self.json_page.raw_json['items'] + self.ChangeRawJsonDependingOnApilibVersion() self.ParseNodes() self.ParseSites() return self.node_dictlist - def ParseResourcesFullSites(self) : - """ UNUSED. Originally used to get information from the sites. - ParseResourcesFull is used instead. + def ParseResourcesFullSites(self): + """ Called by GetSites which is unused. + Originally used to get information from the sites, with for each site + the list of nodes it has, along with their properties. + + :return: site_dict, dictionary of sites + :rtype: dict + + .. warning:: unused + .. seealso:: GetSites (IotlabTestbedAPI) """ - if self.version_json_dict['apilib_version'] != "0.2.10" : - self.json_page.raw_json = self.json_page.raw_json['items'] + self.ChangeRawJsonDependingOnApilibVersion() self.ParseNodes() self.ParseSites() return self.site_dict - def ParseNodes(self): """ Parse nodes properties from OAR Put them into a dictionary with key = node id and value is a dictionary @@ -489,21 +786,23 @@ class OARGETParser: """ node_id = None - keys = self.resources_fulljson_dict.keys() + _resources_fulljson_dict = \ + self.parsing_resourcesfull.resources_fulljson_dict + keys = _resources_fulljson_dict.keys() keys.sort() for dictline in self.json_page.raw_json: node_id = None # dictionary is empty and/or a new node has to be inserted - node_id = self.resources_fulljson_dict['network_address'](\ - self.node_dictlist, dictline['network_address']) + node_id = _resources_fulljson_dict['network_address']( + self.node_dictlist, dictline['network_address']) for k in keys: if k in dictline: if k == 'network_address': continue - self.resources_fulljson_dict[k](\ - self.node_dictlist[node_id], dictline[k]) + _resources_fulljson_dict[k]( + self.node_dictlist[node_id], dictline[k]) #The last property has been inserted in the property tuple list, #reset node_id @@ -512,10 +811,20 @@ class OARGETParser: node_id = None @staticmethod - def iotlab_hostname_to_hrn( root_auth, hostname): - return root_auth + '.'+ hostname + def iotlab_hostname_to_hrn(root_auth, hostname): + """ + Transforms a node hostname into a SFA hrn. + :param root_auth: Name of the root authority of the SFA server. In + our case, it is set to iotlab. + :param hostname: node's hotname, given by OAR. + :type root_auth: string + :type hostname: string + :returns: inserts the root_auth and '.' before the hostname. + :rtype: string + """ + return root_auth + '.' + hostname def ParseSites(self): """ Returns a list of dictionnaries containing the sites' attributes.""" @@ -526,7 +835,7 @@ class OARGETParser: #%(self.node_dictlist)) # Create a list of nodes per site_id for node_id in self.node_dictlist: - node = self.node_dictlist[node_id] + node = self.node_dictlist[node_id] if node['site'] not in nodes_per_site: nodes_per_site[node['site']] = [] @@ -535,112 +844,110 @@ class OARGETParser: if node['node_id'] not in nodes_per_site[node['site']]: nodes_per_site[node['site']].append(node['node_id']) - #Create a site dictionary whose key is site_login_base (name of the site) - # and value is a dictionary of properties, including the list - #of the node_ids + #Create a site dictionary whose key is site_login_base + # (name of the site) and value is a dictionary of properties, + # including the list of the node_ids for node_id in self.node_dictlist: - node = self.node_dictlist[node_id] - #node.update({'hrn':self.iotlab_hostname_to_hrn(self.interface_hrn, \ - #node['site'],node['hostname'])}) - node.update({'hrn':self.iotlab_hostname_to_hrn(self.interface_hrn, node['hostname'])}) - self.node_dictlist.update({node_id:node}) + node = self.node_dictlist[node_id] + node.update({'hrn': self.iotlab_hostname_to_hrn(self.interface_hrn, + node['hostname'])}) + self.node_dictlist.update({node_id: node}) if node['site'] not in self.site_dict: self.site_dict[node['site']] = { - 'site':node['site'], - 'node_ids':nodes_per_site[node['site']], - 'latitude':"48.83726", - 'longitude':"- 2.10336",'name':config.SFA_REGISTRY_ROOT_AUTH, - 'pcu_ids':[], 'max_slices':None, 'ext_consortium_id':None, - 'max_slivers':None, 'is_public':True, 'peer_site_id': None, - 'abbreviated_name':"iotlab", 'address_ids': [], - 'url':"http,//www.senslab.info", 'person_ids':[], - 'site_tag_ids':[], 'enabled': True, 'slice_ids':[], - 'date_created': None, 'peer_id': None } - #if node['site_login_base'] not in self.site_dict.keys(): - #self.site_dict[node['site_login_base']] = {'login_base':node['site_login_base'], - #'node_ids':nodes_per_site[node['site_login_base']], - #'latitude':"48.83726", - #'longitude':"- 2.10336",'name':"senslab", - #'pcu_ids':[], 'max_slices':None, 'ext_consortium_id':None, - #'max_slivers':None, 'is_public':True, 'peer_site_id': None, - #'abbreviated_name':"senslab", 'address_ids': [], - #'url':"http,//www.senslab.info", 'person_ids':[], - #'site_tag_ids':[], 'enabled': True, 'slice_ids':[], - #'date_created': None, 'peer_id': None } - - - + 'site': node['site'], + 'node_ids': nodes_per_site[node['site']], + 'latitude': "48.83726", + 'longitude': "- 2.10336", + 'name': config.SFA_REGISTRY_ROOT_AUTH, + 'pcu_ids': [], 'max_slices': None, + 'ext_consortium_id': None, + 'max_slivers': None, 'is_public': True, + 'peer_site_id': None, + 'abbreviated_name': "iotlab", 'address_ids': [], + 'url': "https://portal.senslab.info", 'person_ids': [], + 'site_tag_ids': [], 'enabled': True, 'slice_ids': [], + 'date_created': None, 'peer_id': None + } OARrequests_uri_dict = { 'GET_version': - {'uri':'/oarapi/version.json', 'parse_func': ParseVersion}, + {'uri': '/oarapi/version.json', 'parse_func': ParseVersion}, + 'GET_timezone': - {'uri':'/oarapi/timezone.json' ,'parse_func': ParseTimezone }, + {'uri': '/oarapi/timezone.json', 'parse_func': ParseTimezone}, + 'GET_jobs': - {'uri':'/oarapi/jobs.json','parse_func': ParseJobs}, + {'uri': '/oarapi/jobs.json', 'parse_func': ParseJobs}, + 'GET_jobs_id': - {'uri':'/oarapi/jobs/id.json','parse_func': ParseJobsIds}, + {'uri': '/oarapi/jobs/id.json', 'parse_func': ParseJobsIds}, + 'GET_jobs_id_resources': - {'uri':'/oarapi/jobs/id/resources.json',\ - 'parse_func': ParseJobsIdResources}, + {'uri': '/oarapi/jobs/id/resources.json', + 'parse_func': ParseJobsIdResources}, + 'GET_jobs_table': - {'uri':'/oarapi/jobs/table.json','parse_func': ParseJobsTable}, + {'uri': '/oarapi/jobs/table.json', 'parse_func': ParseJobsTable}, + 'GET_jobs_details': - {'uri':'/oarapi/jobs/details.json',\ - 'parse_func': ParseJobsDetails}, - 'GET_reserved_nodes': - {'uri': - '/oarapi/jobs/details.json?state=Running,Waiting,Launching',\ - 'owner':'&user=', - 'parse_func':ParseReservedNodes}, + {'uri': '/oarapi/jobs/details.json', 'parse_func': ParseJobsDetails}, + 'GET_reserved_nodes': + {'uri': + '/oarapi/jobs/details.json?state=Running,Waiting,Launching', + 'owner': '&user=', 'parse_func': ParseReservedNodes}, 'GET_running_jobs': - {'uri':'/oarapi/jobs/details.json?state=Running',\ - 'parse_func':ParseRunningJobs}, + {'uri': '/oarapi/jobs/details.json?state=Running', + 'parse_func': ParseRunningJobs}, + 'GET_resources_full': - {'uri':'/oarapi/resources/full.json',\ - 'parse_func': ParseResourcesFull}, + {'uri': '/oarapi/resources/full.json', + 'parse_func': ParseResourcesFull}, + 'GET_sites': - {'uri':'/oarapi/resources/full.json',\ - 'parse_func': ParseResourcesFullSites}, - 'GET_resources': - {'uri':'/oarapi/resources.json' ,'parse_func': ParseResources}, - 'DELETE_jobs_id': - {'uri':'/oarapi/jobs/id.json' ,'parse_func': ParseDeleteJobs} - } + {'uri': '/oarapi/resources/full.json', + 'parse_func': ParseResourcesFullSites}, + 'GET_resources': + {'uri': '/oarapi/resources.json', 'parse_func': ParseResources}, + 'DELETE_jobs_id': + {'uri': '/oarapi/jobs/id.json', 'parse_func': ParseDeleteJobs}} - def SendRequest(self, request, strval = None , username = None): + def SendRequest(self, request, strval=None, username=None): """ Connects to OAR , sends the valid GET requests and uses the appropriate json parsing functions. + :returns: calls to the appropriate parsing function, associated with the + GET request + :rtype: depends on the parsing function called. + + .. seealso:: OARrequests_uri_dict """ save_json = None self.json_page.ResetNextPage() save_json = [] - if request in self.OARrequests_uri_dict : + if request in self.OARrequests_uri_dict: while self.json_page.next_page: - self.json_page.raw_json = self.server.GETRequestToOARRestAPI(\ - request, \ - strval, \ - self.json_page.next_offset, \ - username) + self.json_page.raw_json = self.server.GETRequestToOARRestAPI( + request, + strval, + self.json_page.next_offset, + username) self.json_page.FindNextPage() if self.json_page.concatenate: save_json.append(self.json_page.raw_json) - if self.json_page.concatenate and self.json_page.end : + if self.json_page.concatenate and self.json_page.end: self.json_page.raw_json = \ self.json_page.ConcatenateJsonPages(save_json) return self.OARrequests_uri_dict[request]['parse_func'](self) else: - logger.error("OARRESTAPI OARGetParse __init__ : ERROR_REQUEST " \ - %(request)) - + logger.error("OARRESTAPI OARGetParse __init__ : ERROR_REQUEST " + % (request))