use root tenant when deleting instance if sliver tenant no longer exists
[plstackapi.git] / planetstack / openstack / observer.py
index 037c1c5..d8c3c61 100644 (file)
@@ -37,6 +37,7 @@ class OpenStackObserver:
             return
         while True:
             try:
+                start_time=time.time()
                 logger.info('Observer run loop')
                 #self.sync_roles()
 
@@ -46,6 +47,8 @@ class OpenStackObserver:
                 except:
                     logger.log_exc("Exception in sync_tenants")
                     traceback.print_exc()
+                finish_time = time.time()
+                logger.info('Sync tenants took %f seconds'%(finish_time-start_time))
 
                 logger.info('Calling sync users')
                 try:
@@ -53,6 +56,8 @@ class OpenStackObserver:
                 except:
                     logger.log_exc("Exception in sync_users")
                     traceback.print_exc()
+                finish_time = time.time()
+                logger.info('Sync users took %f seconds'%(finish_time-start_time))
 
                 logger.info('Calling sync tenant roles')
                 try:
@@ -67,6 +72,8 @@ class OpenStackObserver:
                 except:
                     logger.log_exc("Exception in sync slivers")
                     traceback.print_exc()
+                finish_time = time.time()
+                logger.info('Sync slivers took %f seconds'%(finish_time-start_time))
 
                 logger.info('Calling sync sliver ips')
                 try:
@@ -74,6 +81,8 @@ class OpenStackObserver:
                 except:
                     logger.log_exc("Exception in sync_sliver_ips")
                     traceback.print_exc()
+                finish_time = time.time()
+                logger.info('Sync sliver ips took %f seconds'%(finish_time-start_time))
 
                 logger.info('Calling sync networks')
                 try:
@@ -81,6 +90,17 @@ class OpenStackObserver:
                 except:
                     logger.log_exc("Exception in sync_networks")
                     traceback.print_exc()
+                finish_time = time.time()
+                logger.info('Sync networks took %f seconds'%(finish_time-start_time))
+
+                logger.info('Calling sync network slivers')
+                try:
+                    self.sync_network_slivers()
+                except:
+                    logger.log_exc("Exception in sync_network_slivers")
+                    traceback.print_exc()
+                finish_time = time.time()
+                logger.info('Sync network sliver ips took %f seconds'%(finish_time-start_time))
 
                 logger.info('Calling sync external routes')
                 try:
@@ -88,10 +108,19 @@ class OpenStackObserver:
                 except:
                      logger.log_exc("Exception in sync_external_routes")
                      traceback.print_exc()
+                finish_time = time.time()
+                logger.info('Sync external routes took %f seconds'%(finish_time-start_time))
 
                 logger.info('Waiting for event')
+                tBeforeWait = time.time()
                 self.wait_for_event(timeout=300)
-                time.sleep(300)
+
+                # Enforce 5 minutes between wakeups
+                tSleep = 300 - (time.time() - tBeforeWait)
+                if tSleep > 0:
+                    logger.info('Sleeping for %d seconds' % tSleep)
+                    time.sleep(tSleep)
+
                 logger.info('Observer woken up')
             except:
                 logger.log_exc("Exception in observer run loop")
@@ -310,16 +339,21 @@ class OpenStackObserver:
             sliver_dict[sliver.instance_id] = sliver
 
         # delete sliver that don't have a sliver record
-        ctx = self.manager.driver.shell.nova_db.ctx 
+        ctx = self.manager.driver.shell.nova_db.ctx
         instances = self.manager.driver.shell.nova_db.instance_get_all(ctx)
         for instance in instances:
             if instance.uuid not in sliver_dict:
                 try:
-                    # lookup tenant and update context  
-                    tenant = self.manager.driver.shell.keystone.tenants.find(id=instance.project_id) 
-                    self.manager.init_admin(tenant=tenant.name)  
+                    # lookup tenant and update context
+                    try:
+                        tenant = self.manager.driver.shell.keystone.tenants.find(id=instance.project_id)
+                        tenant_name = tenant.name
+                    except:
+                        tenant_name = None
+                        logger.info("exception while retrieving tenant %s. Deleting instance using root tenant." % instance.project_id)
+                    self.manager.init_admin(tenant=tenant_name)
                     self.manager.driver.destroy_instance(instance.uuid)
-                    logger.info("destroyed sliver: %s" % (instance))
+                    logger.info("destroyed sliver: %s" % (instance.uuid))
                 except:
                     logger.log_exc("destroy sliver failed: %s" % instance) 
                 
@@ -350,6 +384,72 @@ class OpenStackObserver:
             except:
                 logger.log_exc("failed to add external route for subnet %s" % subnet)
 
+    def sync_network_slivers(self):
+        networkSlivers = NetworkSliver.objects.all()
+        networkSlivers_by_id = {}
+        networkSlivers_by_port = {}
+        for networkSliver in networkSlivers:
+            networkSlivers_by_id[networkSliver.id] = networkSliver
+            networkSlivers_by_port[networkSliver.port_id] = networkSliver
+
+        networks = Network.objects.all()
+        networks_by_id = {}
+        for network in networks:
+            networks_by_id[network.network_id] = network
+
+        slivers = Sliver.objects.all()
+        slivers_by_instance_id = {}
+        for sliver in slivers:
+            slivers_by_instance_id[sliver.instance_id] = sliver
+
+        ports = self.manager.driver.shell.quantum.list_ports()["ports"]
+        for port in ports:
+            if port["id"] in networkSlivers_by_port:
+                # we already have it
+                print "already accounted for port", port["id"]
+                continue
+
+            if port["device_owner"] != "compute:nova":
+                # we only want the ports that connect to instances
+                continue
+
+            network = networks_by_id.get(port['network_id'], None)
+            if not network:
+                #print "no network for port", port["id"], "network", port["network_id"]
+                continue
+
+            sliver = slivers_by_instance_id.get(port['device_id'], None)
+            if not sliver:
+                print "no sliver for port", port["id"], "device_id", port['device_id']
+                continue
+
+            if network.template.sharedNetworkId is not None:
+                # If it's a shared network template, then more than one network
+                # object maps to the quantum network. We have to do a whole bunch
+                # of extra work to find the right one.
+                networks = network.template.network_set.all()
+                network = None
+                for candidate_network in networks:
+                    if (candidate_network.owner == sliver.slice):
+                        print "found network", candidate_network
+                        network = candidate_network
+
+                if not network:
+                    print "failed to find the correct network for a shared template for port", port["id"], "network", port["network_id"]
+                    continue
+
+            if not port["fixed_ips"]:
+                print "port", port["id"], "has no fixed_ips"
+                continue
+
+#            print "XXX", port
+
+            ns = NetworkSliver(network=network,
+                               sliver=sliver,
+                               ip=port["fixed_ips"][0]["ip_address"],
+                               port_id=port["id"])
+            ns.save()
+
     def sync_networks(self):
         """
         save all networks where enacted < updated or enacted == None. Remove networks that