From: Sapan Bhatia Date: Wed, 16 Jul 2014 04:17:33 +0000 (-0400) Subject: Merge branch 'master' of ssh://git.planet-lab.org/git/plstackapi into observer3.0 X-Git-Url: http://git.onelab.eu/?p=plstackapi.git;a=commitdiff_plain;h=f0ecead28a78eec4e395c206120b99e4aebec1e7 Merge branch 'master' of ssh://git.planet-lab.org/git/plstackapi into observer3.0 Conflicts: planetstack/core/fixtures/demo_data.json planetstack/core/fixtures/initial_data.json planetstack/core/models/plcorebase.py planetstack/observer/event_loop.py planetstack/observer/event_manager.py planetstack/observer/steps/garbage_collector.py planetstack/observer/syncstep.py --- f0ecead28a78eec4e395c206120b99e4aebec1e7 diff --cc planetstack/core/models/plcorebase.py index d4932a0,ec79419..34af4a4 --- a/planetstack/core/models/plcorebase.py +++ b/planetstack/core/models/plcorebase.py @@@ -13,27 -15,19 +15,32 @@@ except traceback.print_exc() # guard against something failing - def notify_observer(): + def notify_observer(*args, **kwargs): pass -class PlCoreBase(models.Model): +# This manager will be inherited by all subclasses because +# the core model is abstract. +class PlCoreBaseManager(models.Manager): + def get_query_set(self): + return super(PlCoreBaseManager, self).get_query_set().filter(deleted=False) +class PlCoreBase(models.Model): + objects = PlCoreBaseManager() + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + # default values for created and updated are only there to keep evolution + # from failing. + + created = models.DateTimeField(auto_now_add=True, default=datetime.datetime.now()) + updated = models.DateTimeField(auto_now=True, default=datetime.datetime.now()) enacted = models.DateTimeField(null=True, default=None) + backend_status = models.CharField(max_length=140, + default="Provisioning in progress") + deleted = models.BooleanField(default=False) class Meta: + # Changing abstract to False would require the managers of subclasses of + # PlCoreBase to be customized individually. abstract = True app_label = "core" diff --cc planetstack/observer/event_loop.py index d659ab5,6c19215..ec49dd7 --- a/planetstack/observer/event_loop.py +++ b/planetstack/observer/event_loop.py @@@ -16,10 -16,9 +16,10 @@@ from openstack.driver import OpenStackD from util.logger import Logger, logging, logger #from timeout import timeout from planetstack.config import Config - from observer.steps import * + #from observer.steps import * from syncstep import SyncStep from toposort import toposort +from observer.error_mapper import error_mapper debug_mode = False diff --cc planetstack/observer/event_manager.py index 19d9e25,bd04ced..d2a53a7 --- a/planetstack/observer/event_manager.py +++ b/planetstack/observer/event_manager.py @@@ -12,83 -13,149 +12,149 @@@ from fofum import Fofu import json import traceback - # decorator that marks dispatachable event methods - def event(func): - setattr(func, 'event', func.__name__) - return func - - class EventHandler: - # This code is currently not in use. - def __init__(self): - pass - - @staticmethod - def get_events(): - events = [] - for name in dir(EventHandler): - attribute = getattr(EventHandler, name) - if hasattr(attribute, 'event'): - events.append(getattr(attribute, 'event')) - return events + random_client_id=None + def get_random_client_id(): + global random_client_id - def dispatch(self, event, *args, **kwds): - if hasattr(self, event): - return getattr(self, event)(*args, **kwds) - - - class EventSender: - def __init__(self,user=None,clientid=None): + if (random_client_id is None) and os.path.exists("/opt/planetstack/random_client_id"): + # try to use the last one we used, if we saved it try: - user = Config().feefie_client_user + random_client_id = open("/opt/planetstack/random_client_id","r").readline().strip() + print "get_random_client_id: loaded %s" % random_client_id except: - user = 'pl' + print "get_random_client_id: failed to read /opt/planetstack/random_client_id" + if random_client_id is None: + random_client_id = base64.urlsafe_b64encode(os.urandom(12)) + print "get_random_client_id: generated new id %s" % random_client_id + + # try to save it for later (XXX: could race with another client here) try: - clid = Config().feefie_client_id + open("/opt/planetstack/random_client_id","w").write("%s\n" % random_client_id) except: - clid = self.random_client_id() - + print "get_random_client_id: failed to write /opt/planetstack/random_client_id" + + return random_client_id + + # decorator that marks dispatachable event methods + def event(func): + setattr(func, 'event', func.__name__) + return func - self.fofum = Fofum(user=user) - self.fofum.make(clid) + class EventHandler: + # This code is currently not in use. + def __init__(self): + pass + + @staticmethod + def get_events(): + events = [] + for name in dir(EventHandler): + attribute = getattr(EventHandler, name) + if hasattr(attribute, 'event'): + events.append(getattr(attribute, 'event')) + return events + + def dispatch(self, event, *args, **kwds): + if hasattr(self, event): + return getattr(self, event)(*args, **kwds) + - def fire(self,**kwargs): + class EventSender: + def __init__(self,user=None,clientid=None): + try: + user = Config().feefie_client_user + except: + user = 'pl' + + try: + clid = Config().feefie_client_id + except: + clid = get_random_client_id() + print "EventSender: no feefie_client_id configured. Using random id %s" % clid + + self.fofum = Fofum(user=user) + self.fofum.make(clid) + + def fire(self,**kwargs): kwargs["uuid"] = str(uuid.uuid1()) - self.fofum.fire(json.dumps(kwargs)) + self.fofum.fire(json.dumps(kwargs)) class EventListener: - def __init__(self,wake_up=None): - self.handler = EventHandler() - self.wake_up = wake_up - - def handle_event(self, payload): - payload_dict = json.loads(payload) - - if (self.wake_up): - self.wake_up() - - def random_client_id(self): - try: - return self.client_id - except AttributeError: - self.client_id = base64.urlsafe_b64encode(os.urandom(12)) - return self.client_id - - def run(self): - # This is our unique client id, to be used when firing and receiving events - # It needs to be generated once and placed in the config file - - try: - user = Config().feefie_client_user - except: - user = 'pl' - - try: - clid = Config().feefie_client_id - except: - clid = self.random_client_id() - - f = Fofum(user=user) - - listener_thread = threading.Thread(target=f.listen_for_event,args=(clid,self.handle_event)) - listener_thread.start() + def __init__(self,wake_up=None): + self.handler = EventHandler() + self.wake_up = wake_up + self.deleters = {} + self.load_deleter_modules() + + def load_deleter_modules(self, deleter_dir=None): + if deleter_dir is None: + if hasattr(Config(), "observer_deleters_dir"): + deleter_dir = Config().observer_deleters_dir + else: + deleter_dir = "/opt/planetstack/observer/deleters" + + for fn in os.listdir(deleter_dir): + pathname = os.path.join(deleter_dir,fn) + if os.path.isfile(pathname) and fn.endswith(".py") and (fn!="__init__.py"): + module = imp.load_source(fn[:-3],pathname) + for classname in dir(module): + c = getattr(module, classname, None) + + # make sure 'c' is a descendent of Deleter and has a + # provides field (this eliminates the abstract base classes + # since they don't have a provides) + + if inspect.isclass(c) and issubclass(c, Deleter) and hasattr(c,"model") and c.model!=None: + modelName = c.model + if not modelName in self.deleters: + self.deleters[modelName] = [] + if not (c in self.deleters[modelName]): + self.deleters[modelName].append(c) + print 'loaded deleters: %s' % ",".join(self.deleters.keys()) + + + def handle_event(self, payload): + payload_dict = json.loads(payload) + + try: + deletion = payload_dict.get('delete_flag', False) + if (deletion): + model = payload_dict['model'] + pk = payload_dict['pk'] + model_dict = payload_dict['model_dict'] + + for deleter in self.deleters[model]: + try: + deleter()(pk, model_dict) + except: + # something is silently eating these + # exceptions... + traceback.print_exc() + raise + + except: + deletion = False + + if (not deletion and self.wake_up): + self.wake_up() + + def run(self): + # This is our unique client id, to be used when firing and receiving events + # It needs to be generated once and placed in the config file + + try: + user = Config().feefie_client_user + except: + user = 'pl' + + try: + clid = Config().feefie_client_id + except: + clid = get_random_client_id() + print "EventListener: no feefie_client_id configured. Using random id %s" % clid + + f = Fofum(user=user) + + listener_thread = threading.Thread(target=f.listen_for_event,args=(clid,self.handle_event)) + listener_thread.start()