Merge branch 'master' of ssh://git.planet-lab.org/git/plstackapi into observer3.0

Conflicts:
	planetstack/core/fixtures/demo_data.json
	planetstack/core/fixtures/initial_data.json
	planetstack/core/models/plcorebase.py
	planetstack/observer/event_loop.py
	planetstack/observer/event_manager.py
	planetstack/observer/steps/garbage_collector.py
	planetstack/observer/syncstep.py

diff --cc planetstack/core/models/plcorebase.py
index d4932a0,ec79419..34af4a4
--- 1/planetstack/core/models/plcorebase.py
--- 2/planetstack/core/models/plcorebase.py
+++ b/planetstack/core/models/plcorebase.py
@@@ -13,27 -15,19 +15,32 @@@ except
      traceback.print_exc()
  
      # guard against something failing
-     def notify_observer():
+     def notify_observer(*args, **kwargs):
          pass
  
 -class PlCoreBase(models.Model):
 +# This manager will be inherited by all subclasses because
 +# the core model is abstract.
 +class PlCoreBaseManager(models.Manager):
 +    def get_query_set(self):
 +        return super(PlCoreBaseManager, self).get_query_set().filter(deleted=False)
  
 +class PlCoreBase(models.Model):
 +    objects = PlCoreBaseManager()
 +    created = models.DateTimeField(auto_now_add=True)
 +    updated = models.DateTimeField(auto_now=True)
+     # default values for created and updated are only there to keep evolution
+     # from failing.
+ 
+     created = models.DateTimeField(auto_now_add=True, default=datetime.datetime.now())
+     updated = models.DateTimeField(auto_now=True, default=datetime.datetime.now())
      enacted = models.DateTimeField(null=True, default=None)
 +    backend_status = models.CharField(max_length=140,
 +                                      default="Provisioning in progress")
 +    deleted = models.BooleanField(default=False)
  
      class Meta:
 +        # Changing abstract to False would require the managers of subclasses of
 +        # PlCoreBase to be customized individually.
          abstract = True
          app_label = "core"
  

diff --cc planetstack/core/models/user.py
Simple merge

diff --cc planetstack/observer/event_loop.py
index d659ab5,6c19215..ec49dd7
--- 1/planetstack/observer/event_loop.py
--- 2/planetstack/observer/event_loop.py
+++ b/planetstack/observer/event_loop.py
@@@ -16,10 -16,9 +16,10 @@@ from openstack.driver import OpenStackD
  from util.logger import Logger, logging, logger
  #from timeout import timeout
  from planetstack.config import Config
- from observer.steps import *
+ #from observer.steps import *
  from syncstep import SyncStep
  from toposort import toposort
 +from observer.error_mapper import error_mapper
  
  debug_mode = False
  

diff --cc planetstack/observer/event_manager.py
index 19d9e25,bd04ced..d2a53a7
--- 1/planetstack/observer/event_manager.py
--- 2/planetstack/observer/event_manager.py
+++ b/planetstack/observer/event_manager.py
@@@ -12,83 -13,149 +12,149 @@@ from fofum import Fofu
  import json
  import traceback
  
- # decorator that marks dispatachable event methods    
- def event(func):
-     setattr(func, 'event', func.__name__)
-     return func         
- 
- class EventHandler:
-     # This code is currently not in use.
-     def __init__(self):
-         pass 
- 
-     @staticmethod
-     def get_events():
-         events = []
-         for name in dir(EventHandler):
-             attribute = getattr(EventHandler, name)
-             if hasattr(attribute, 'event'):
-                 events.append(getattr(attribute, 'event'))
-         return events
+ random_client_id=None
+ def get_random_client_id():
+     global random_client_id
  
-     def dispatch(self, event, *args, **kwds):
-         if hasattr(self, event):
-             return getattr(self, event)(*args, **kwds)
- 
- 
- class EventSender:
-     def __init__(self,user=None,clientid=None):
+     if (random_client_id is None) and os.path.exists("/opt/planetstack/random_client_id"):
+         # try to use the last one we used, if we saved it
          try:
-             user = Config().feefie_client_user
+             random_client_id = open("/opt/planetstack/random_client_id","r").readline().strip()
+             print "get_random_client_id: loaded %s" % random_client_id
          except:
-             user = 'pl'
+             print "get_random_client_id: failed to read /opt/planetstack/random_client_id"
  
+     if random_client_id is None:
+         random_client_id = base64.urlsafe_b64encode(os.urandom(12))
+         print "get_random_client_id: generated new id %s" % random_client_id
+ 
+         # try to save it for later (XXX: could race with another client here)
          try:
-             clid = Config().feefie_client_id
+             open("/opt/planetstack/random_client_id","w").write("%s\n" % random_client_id)
          except:
-             clid = self.random_client_id()
-             
+             print "get_random_client_id: failed to write /opt/planetstack/random_client_id"
+ 
+     return random_client_id
+ 
+ # decorator that marks dispatachable event methods
+ def event(func):
+       setattr(func, 'event', func.__name__)
+       return func
  
-         self.fofum = Fofum(user=user)
-         self.fofum.make(clid)
+ class EventHandler:
+       # This code is currently not in use.
+       def __init__(self):
+               pass
+ 
+       @staticmethod
+       def get_events():
+               events = []
+               for name in dir(EventHandler):
+                       attribute = getattr(EventHandler, name)
+                       if hasattr(attribute, 'event'):
+                               events.append(getattr(attribute, 'event'))
+               return events
+ 
+       def dispatch(self, event, *args, **kwds):
+               if hasattr(self, event):
+                       return getattr(self, event)(*args, **kwds)
+                       
  
-     def fire(self,**kwargs):
+ class EventSender:
+       def __init__(self,user=None,clientid=None):
+               try:
+                       user = Config().feefie_client_user
+               except:
+                       user = 'pl'
+ 
+               try:
+                       clid = Config().feefie_client_id
+               except:
+                       clid = get_random_client_id()
+                         print "EventSender: no feefie_client_id configured. Using random id %s" % clid
+ 
+               self.fofum = Fofum(user=user)
+               self.fofum.make(clid)
+ 
+       def fire(self,**kwargs):
                  kwargs["uuid"] = str(uuid.uuid1())
 -              self.fofum.fire(json.dumps(kwargs))
 +        self.fofum.fire(json.dumps(kwargs))
  
  class EventListener:
-     def __init__(self,wake_up=None):
-         self.handler = EventHandler()
-         self.wake_up = wake_up
- 
-     def handle_event(self, payload):
-         payload_dict = json.loads(payload)
- 
-         if (self.wake_up):
-             self.wake_up()
- 
-     def random_client_id(self):
-         try:
-             return self.client_id
-         except AttributeError:
-             self.client_id = base64.urlsafe_b64encode(os.urandom(12))
-             return self.client_id
- 
-     def run(self):
-         # This is our unique client id, to be used when firing and receiving events
-         # It needs to be generated once and placed in the config file
- 
-         try:
-             user = Config().feefie_client_user
-         except:
-             user = 'pl'
- 
-         try:
-             clid = Config().feefie_client_id
-         except:
-             clid = self.random_client_id()
- 
-         f = Fofum(user=user)
-         
-         listener_thread = threading.Thread(target=f.listen_for_event,args=(clid,self.handle_event))
-         listener_thread.start()
+       def __init__(self,wake_up=None):
+               self.handler = EventHandler()
+               self.wake_up = wake_up
+                 self.deleters = {}
+                 self.load_deleter_modules()
+ 
+         def load_deleter_modules(self, deleter_dir=None):
+             if deleter_dir is None:
+                 if hasattr(Config(), "observer_deleters_dir"):
+                     deleter_dir = Config().observer_deleters_dir
+                 else:
+                     deleter_dir = "/opt/planetstack/observer/deleters"
+ 
+             for fn in os.listdir(deleter_dir):
+                 pathname = os.path.join(deleter_dir,fn)
+                 if os.path.isfile(pathname) and fn.endswith(".py") and (fn!="__init__.py"):
+                     module = imp.load_source(fn[:-3],pathname)
+                     for classname in dir(module):
+                         c = getattr(module, classname, None)
+ 
+                         # make sure 'c' is a descendent of Deleter and has a
+                         # provides field (this eliminates the abstract base classes
+                         # since they don't have a provides)
+ 
+                         if inspect.isclass(c) and issubclass(c, Deleter) and hasattr(c,"model") and c.model!=None:
+                             modelName = c.model
+                             if not modelName in self.deleters:
+                                 self.deleters[modelName] = []
+                             if not (c in self.deleters[modelName]):
+                                 self.deleters[modelName].append(c)
+             print 'loaded deleters: %s' % ",".join(self.deleters.keys())
+ 
+ 
+       def handle_event(self, payload):
+               payload_dict = json.loads(payload)
+ 
+               try:
+                       deletion = payload_dict.get('delete_flag', False)
+                       if (deletion):
+                               model = payload_dict['model']
+                               pk = payload_dict['pk']
+                                 model_dict = payload_dict['model_dict']
+ 
+                               for deleter in self.deleters[model]:
+                                         try:
+                                           deleter()(pk, model_dict)
+                                         except:
+                                             # something is silently eating these
+                                             # exceptions...
+                                             traceback.print_exc()
+                                             raise
+ 
+               except:
+                       deletion = False
+ 
+               if (not deletion and self.wake_up):
+                       self.wake_up()
+ 
+       def run(self):
+               # This is our unique client id, to be used when firing and receiving events
+               # It needs to be generated once and placed in the config file
+ 
+               try:
+                       user = Config().feefie_client_user
+               except:
+                       user = 'pl'
+ 
+               try:
+                       clid = Config().feefie_client_id
+               except:
+                       clid = get_random_client_id()
+                         print "EventListener: no feefie_client_id configured. Using random id %s" % clid
+ 
+               f = Fofum(user=user)
+               
+               listener_thread = threading.Thread(target=f.listen_for_event,args=(clid,self.handle_event))
+               listener_thread.start()

diff --cc planetstack/observer/syncstep.py
Simple merge