Fixing bug in querying OAR.

[sfa.git] / sfa / senslab / OARrestapi.py

diff --git a/sfa/senslab/OARrestapi.py b/sfa/senslab/OARrestapi.py
index ee7f8b4..a59fe0f 100644 (file)
--- a/sfa/senslab/OARrestapi.py
+++ b/sfa/senslab/OARrestapi.py
@@ -3,7 +3,8 @@ from httplib import HTTPConnection, HTTPException
 import json
 #import datetime
 #from time import gmtime, strftime 
-
+import os.path
+import sys
 #import urllib
 #import urllib2
 from sfa.util.config import Config
@@ -12,8 +13,6 @@ from sfa.util.config import Config
 from sfa.util.sfalogging import logger
 
 
-OARIP = '194.199.16.166'
-
 OAR_REQUEST_POST_URI_DICT = {'POST_job':{'uri': '/oarapi/jobs.json'},
                             'DELETE_jobs_id':{'uri':'/oarapi/jobs/id.json'},
                             }
@@ -28,14 +27,26 @@ POST_FORMAT = {'json' : {'content':"application/json", 'object':json},}
 
 
 class OARrestapi:
-    def __init__(self):
+    def __init__(self, config_file =  '/etc/sfa/oar_config.py'):
         self.oarserver = {}
-        self.oarserver['ip'] = OARIP
-        self.oarserver['port'] = 8800
+       
+        
         self.oarserver['uri'] = None
         self.oarserver['postformat'] = 'json'
+        
+        try:
+            execfile(config_file, self.__dict__)
+       
+            self.config_file = config_file
+            # path to configuration data
+            self.config_path = os.path.dirname(config_file)
+            
+        except IOError:
+            raise IOError, "Could not find or load the configuration file: %s" \
+                            % config_file
         #logger.setLevelDebug()
-
+        self.oarserver['ip'] = self.OAR_IP
+        self.oarserver['port'] = self.OAR_PORT
         self.jobstates  = ['Terminated', 'Hold', 'Waiting', 'toLaunch', \
                             'toError', 'toAckReservation', 'Launching', \
                             'Finishing', 'Running', 'Suspended', 'Resuming',\
@@ -44,7 +55,7 @@ class OARrestapi:
         self.parser = OARGETParser(self)
        
             
-    def GETRequestToOARRestAPI(self, request, strval=None , username = None ): 
+    def GETRequestToOARRestAPI(self, request, strval=None ,next_page=None, username = None ): 
         self.oarserver['uri'] = \
                             OARGETParser.OARrequests_uri_dict[request]['uri']
         #Get job details with username                   
@@ -56,11 +67,17 @@ class OARrestapi:
         if strval:
             self.oarserver['uri'] = self.oarserver['uri'].\
                                             replace("id",str(strval))
-            logger.debug("OARrestapi: \t  GETRequestToOARRestAPI  \
-                            self.oarserver['uri'] %s strval %s" \
-                            %(self.oarserver['uri'], strval))
+            
+        if next_page:
+            self.oarserver['uri'] += next_page
+            
         if username:
             headers['X-REMOTE_IDENT'] = username 
+            
+        print>>sys.stderr, " \r\n \t    OARrestapi \tGETRequestToOARRestAPI %s" %( self.oarserver['uri'])
+        logger.debug("OARrestapi: \t  GETRequestToOARRestAPI  \
+                        self.oarserver['uri'] %s strval %s" \
+                        %(self.oarserver['uri'], strval))
         try :  
             #seems that it does not work if we don't add this
             headers['content-length'] = '0' 
@@ -76,6 +93,7 @@ class OARrestapi:
             #raise ServerError("GET_OAR_SRVR : Could not reach OARserver")
         try:
             js_dict = json.loads(resp)
+            #print "\r\n \t\t\t js_dict keys" , js_dict.keys(), " \r\n", js_dict
             return js_dict
         
         except ValueError, error:
@@ -530,16 +548,62 @@ class OARGETParser:
         }
 
 
-
+    def FindNextPage(self):
+        if "links" in self.raw_json:
+            for page in self.raw_json['links']:
+                if page['rel'] == 'next':
+                    self.concatenate = True
+                    print>>sys.stderr, " \r\n \t\t FindNextPage  self.concatenate %s" %(self.concatenate )
+                    return True, "?"+page['href'].split("?")[1]
+        if self.concatenate :
+            self.end = True
+            print>>sys.stderr, " \r\n \t\t END FindNextPage  self.concatenate %s" %(self.concatenate )
+        return False, None
+            
+    def ConcatenateJsonPages (self, saved_json_list):
+        #reset items list
+
+        tmp = {}
+        tmp['items'] = []
+        print >>sys.stderr, " \r\n ConcatenateJsonPages saved_json_list len ", len(saved_json_list)
+        for page in saved_json_list:
+            #for node in page['items']:
+                #self.raw_json['items'].append(node)
+            print>>sys.stderr, " \r\n ConcatenateJsonPages  page['items']len ", len(page['items'])
+            tmp['items'].extend(page['items'])
+        #print>>sys.stderr, " \r\n ConcatenateJsonPages len ", len(self.raw_json['items'])
+        #print>>sys.stderr, " \r\n ConcatenateJsonPages  self.raw_json['items']", self.raw_json['items']        
+        return tmp
+                        
     def SendRequest(self, request, strval = None , username = None):
         """ Connects to OAR , sends the valid GET requests and uses
         the appropriate json parsing functions.
         
         """
+        self.raw_json = None
+        next_page = True
+        next_offset = None
+        save_json = None
+        self.concatenate = False
+        self.end = False
+        a = 0
+        save_json = []
+        self.raw_json_list = []
         if request in self.OARrequests_uri_dict :
-            self.raw_json = self.server.GETRequestToOARRestAPI(request, \
+            while next_page:
+                self.raw_json = self.server.GETRequestToOARRestAPI(request, \
                                                                 strval, \
-                                                                username) 
+                                                                next_offset, \
+                                                                username)
+                
+                next_page , next_offset = self.FindNextPage()
+                if self.concatenate:
+                    #self.raw_json_list.append(self.raw_json)
+                    save_json.append(self.raw_json)
+            if self.concatenate and self.end :
+                #self.raw_json = self.ConcatenateJsonPages(self.raw_json_list) 
+                self.raw_json = self.ConcatenateJsonPages(save_json)
+
             return self.OARrequests_uri_dict[request]['parse_func'](self)
         else:
             logger.error("OARRESTAPI OARGetParse __init__ : ERROR_REQUEST " \