import time
import traceback
+import commands
+import threading
+
from datetime import datetime
from collections import defaultdict
from core.models import *
from django.db.models import F, Q
from openstack.manager import OpenStackManager
-from util.logger import Logger, logging
+from util.logger import Logger, logging, logger
+#from timeout import timeout
logger = Logger(logfile='observer.log', level=logging.INFO)
def __init__(self):
self.manager = OpenStackManager()
+ # The Condition object that gets signalled by Feefie events
+ self.event_cond = threading.Condition()
+
+ def wait_for_event(self, timeout):
+ self.event_cond.acquire()
+ self.event_cond.wait(timeout)
+ self.event_cond.release()
+
+ def wake_up(self):
+ logger.info('Wake up routine called. Event cond %r'%self.event_cond)
+ self.event_cond.acquire()
+ self.event_cond.notify()
+ self.event_cond.release()
def run(self):
if not self.manager.enabled or not self.manager.has_openstack:
return
while True:
try:
+ logger.info('Observer run loop')
#self.sync_roles()
- self.sync_tenants()
- self.sync_users()
- self.sync_user_tenant_roles()
- self.sync_slivers()
- self.sync_sliver_ips()
- time.sleep(7)
+
+ logger.info('Calling sync tenants')
+ try:
+ self.sync_tenants()
+ except:
+ logger.log_exc("Exception in sync_tenants")
+ traceback.print_exc()
+
+ logger.info('Calling sync users')
+ try:
+ self.sync_users()
+ except:
+ logger.log_exc("Exception in sync_users")
+ traceback.print_exc()
+
+ logger.info('Calling sync tenant roles')
+ try:
+ self.sync_user_tenant_roles()
+ except:
+ logger.log_exc("Exception in sync_users")
+ traceback.print_exc()
+
+ logger.info('Calling sync slivers')
+ try:
+ self.sync_slivers()
+ except:
+ logger.log_exc("Exception in sync slivers")
+ traceback.print_exc()
+
+ logger.info('Calling sync sliver ips')
+ try:
+ self.sync_sliver_ips()
+ except:
+ logger.log_exc("Exception in sync_sliver_ips")
+ traceback.print_exc()
+
+ logger.info('Calling sync networks')
+ try:
+ self.sync_networks()
+ except:
+ logger.log_exc("Exception in sync_networks")
+ traceback.print_exc()
+
+ logger.info('Calling sync external routes')
+ try:
+ self.sync_external_routes()
+ except:
+ logger.log_exc("Exception in sync_external_routes")
+ traceback.print_exc()
+
+ logger.info('Waiting for event')
+ tBeforeWait = time.time()
+ self.wait_for_event(timeout=300)
+
+ # Enforce 5 minutes between wakeups
+ tSleep = 300 - (time.time() - tBeforeWait)
+ if tSleep > 0:
+ logger.info('Sleeping for %d seconds' % tSleep)
+ time.sleep(tSleep)
+
+ logger.info('Observer woken up')
except:
- traceback.print_exc()
+ logger.log_exc("Exception in observer run loop")
+ traceback.print_exc()
def sync_roles(self):
"""
tenants = self.manager.driver.shell.keystone.tenants.findall()
system_tenants = ['admin','service']
for tenant in tenants:
- if tenant.name == 'admin':
+ if tenant.name in system_tenants:
continue
if tenant.name not in site_dict and tenant.name not in slice_dict:
try:
# get all users that need to be synced (enacted < updated or enacted is None)
pending_slivers = Sliver.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
for sliver in pending_slivers:
- if not sliver.instance_id and sliver.creator:
+ if sliver.creator:
try:
# update manager context
self.manager.init_caller(sliver.creator, sliver.slice.name)
self.manager.save_sliver(sliver)
- logger.info("saved sliver: %s %s" % (sliver))
+ logger.info("saved sliver: %s" % (sliver))
except:
logger.log_exc("save sliver failed: %s" % sliver)
- # get all slivers that where enacted != null. We can assume these users
+ # get all slivers where enacted != null. We can assume these users
# have previously been synced and need to be checed for deletion.
slivers = Sliver.objects.filter(enacted__isnull=False)
sliver_dict = {}
ctx = self.manager.driver.shell.nova_db.ctx
instances = self.manager.driver.shell.nova_db.instance_get_all(ctx)
for instance in instances:
- if instance.id not in sliver_dict:
+ if instance.uuid not in sliver_dict:
try:
# lookup tenant and update context
tenant = self.manager.driver.shell.keystone.tenants.find(id=instance.project_id)
self.manager.init_admin(tenant=tenant.name)
- self.manager.driver.destroy_instance(instance.id)
+ self.manager.driver.destroy_instance(instance.uuid)
logger.info("destroyed sliver: %s" % (instance))
except:
logger.log_exc("destroy sliver failed: %s" % instance)
for sliver in slivers:
# update connection
self.manager.init_admin(tenant=sliver.slice.name)
- servers = self.manager.client.nova.servers.findall(id=sliver.instance_id)
+ servers = self.manager.driver.shell.nova.servers.findall(id=sliver.instance_id)
if not servers:
continue
server = servers[0]
sliver.ip = ips[0]['addr']
sliver.save()
logger.info("saved sliver ip: %s %s" % (sliver, ips[0]))
+
+ def sync_external_routes(self):
+ routes = self.manager.driver.get_external_routes()
+ subnets = self.manager.driver.shell.quantum.list_subnets()['subnets']
+ for subnet in subnets:
+ try:
+ self.manager.driver.add_external_route(subnet, routes)
+ except:
+ logger.log_exc("failed to add external route for subnet %s" % subnet)
+
+ def sync_networks(self):
+ """
+ save all networks where enacted < updated or enacted == None. Remove networks that
+ no don't exist in openstack db if they have an enacted time (enacted != None).
+ """
+ # get all users that need to be synced (enacted < updated or enacted is None)
+ pending_networks = Network.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
+ for network in pending_networks:
+ if network.owner and network.owner.creator:
+ try:
+ # update manager context
+ self.manager.init_caller(network.owner.creator, network.owner.name)
+ self.manager.save_network(network)
+ logger.info("saved network: %s" % (network))
+ except:
+ logger.log_exc("save network failed: %s" % network)
+
+ # get all networks where enacted != null. We can assume these users
+ # have previously been synced and need to be checed for deletion.
+ networks = Network.objects.filter(enacted__isnull=False)
+ network_dict = {}
+ for network in networks:
+ network_dict[network.network_id] = network
+
+ # TODO: delete Network objects if quantum network doesn't exist
+ # (need to write self.manager.driver.shell.quantum_db)
+
git://git.onelab.eu / plstackapi.git / blobdiff