From 511ea5f61a5db72e000c801789bf5f88e8c7009a Mon Sep 17 00:00:00 2001 From: Sapan Bhatia Date: Mon, 21 Jul 2014 22:53:58 -0400 Subject: [PATCH] This tree was mixed up, with an old version of the EC2 Observer. This change copies across the version demo'd the OpenCloud developer meeting. --- planetstack/ec2_observer/steps/__init__.py | 26 +-- .../ec2_observer/steps/sync_deployments.py | 22 +++ planetstack/ec2_observer/steps/sync_images.py | 53 ++++-- planetstack/ec2_observer/steps/sync_nodes.py | 48 ++++++ .../steps/sync_site_deployments.py | 63 ++++--- planetstack/ec2_observer/steps/sync_sites.py | 29 ++-- .../ec2_observer/steps/sync_slivers.py | 158 ++++++++++-------- planetstack/ec2_observer/steps/sync_users.py | 49 ++++-- 8 files changed, 304 insertions(+), 144 deletions(-) create mode 100644 planetstack/ec2_observer/steps/sync_deployments.py create mode 100644 planetstack/ec2_observer/steps/sync_nodes.py diff --git a/planetstack/ec2_observer/steps/__init__.py b/planetstack/ec2_observer/steps/__init__.py index eabf46c..e56c4b1 100644 --- a/planetstack/ec2_observer/steps/__init__.py +++ b/planetstack/ec2_observer/steps/__init__.py @@ -1,15 +1,15 @@ #from .sync_external_routes import SyncExternalRoutes -from .sync_network_slivers import SyncNetworkSlivers -from .sync_networks import SyncNetworks -from .sync_network_deployments import SyncNetworkDeployments -from .sync_site_privileges import SyncSitePrivileges -from .sync_sites import SyncSites -from .sync_slice_memberships import SyncSliceMemberships -from .sync_slices import SyncSlices +#from .sync_network_slivers import SyncNetworkSlivers +#from .sync_networks import SyncNetworks +#from .sync_network_deployments import SyncNetworkDeployments +#from .sync_site_privileges import SyncSitePrivileges +#from .sync_slice_memberships import SyncSliceMemberships +#from .sync_slices import SyncSlices #from .sync_sliver_ips import SyncSliverIps -from .sync_slivers import SyncSlivers -from .sync_users import SyncUsers -from .sync_roles import SyncRoles -from .sync_nodes import SyncNodes -from .sync_images import SyncImages -from .garbage_collector import GarbageCollector +#from .sync_slivers import SyncSlivers +#from .sync_users import SyncUsers +#from .sync_roles import SyncRoles +#from .sync_nodes import SyncNodes +#from .sync_images import SyncImages +#from .garbage_collector import GarbageCollector + diff --git a/planetstack/ec2_observer/steps/sync_deployments.py b/planetstack/ec2_observer/steps/sync_deployments.py new file mode 100644 index 0000000..8a258f1 --- /dev/null +++ b/planetstack/ec2_observer/steps/sync_deployments.py @@ -0,0 +1,22 @@ +import os +import base64 +from django.db.models import F, Q +from planetstack.config import Config +from ec2_observer.syncstep import SyncStep +from core.models.site import * + +class SyncDeployments(SyncStep): + requested_interval=86400 + provides=[Deployment] + + def fetch_pending(self,deletion): + deployments = Deployment.objects.filter(Q(name="Amazon EC2")) + if (not deployments): + deployments = [Deployment(name="Amazon EC2")] + else: + deployments = [] + + return deployments + + def sync_record(self, deployment): + deployment.save() diff --git a/planetstack/ec2_observer/steps/sync_images.py b/planetstack/ec2_observer/steps/sync_images.py index 32b3363..3a15b33 100644 --- a/planetstack/ec2_observer/steps/sync_images.py +++ b/planetstack/ec2_observer/steps/sync_images.py @@ -2,30 +2,59 @@ import os import base64 from django.db.models import F, Q from planetstack.config import Config -from observer.syncstep import SyncStep +from ec2_observer.syncstep import SyncStep from core.models.image import Image -from awslib import * +from ec2_observer.awslib import * -class SyncImages(OpenStackSyncStep): + +class SyncImages(SyncStep): provides=[Image] requested_interval=3600 - def fetch_pending(self): + def fetch_pending(self,deletion): images = Image.objects.all() image_names = [image.name for image in images] new_images = [] - aws_images = aws_run('ec2 describe-images') + try: + aws_images = json.loads(open('/opt/planetstack/aws-images').read()) + except: + aws_images = aws_run('ec2 describe-images --owner 099720109477') + open('/opt/planetstack/aws-images','w').write(json.dumps(aws_images)) + + + + aws_images=aws_images['Images'] + aws_images=filter(lambda x:x['ImageType']=='machine',aws_images)[:50] + names = set([]) for aws_image in aws_images: - if aws_image not in image_names: - image = Image(image_id=image_id, - name=aws_image['name'], - disk_format='XXX' - container_format='XXX' - new_images.append(image) - + desc_ok = True + + try: + desc = aws_image['Description'] + except: + try: + desc = aws_image['ImageLocation'] + except: + desc_ok = False + + if (desc_ok): + try: + desc_ok = desc and desc not in names and desc not in image_names and '14.04' in desc + except KeyError: + desc_ok = False + + if desc_ok and aws_image['ImageType']=='machine': + image = Image(disk_format=aws_image['ImageLocation'], + name=desc, + container_format=aws_image['Hypervisor'], + path=aws_image['ImageId']) + new_images.append(image) + names.add(image.name) + + #print new_images return new_images def sync_record(self, image): diff --git a/planetstack/ec2_observer/steps/sync_nodes.py b/planetstack/ec2_observer/steps/sync_nodes.py new file mode 100644 index 0000000..b66eb2e --- /dev/null +++ b/planetstack/ec2_observer/steps/sync_nodes.py @@ -0,0 +1,48 @@ +import os +import base64 +import random +import time +from datetime import datetime +from django.db.models import F, Q +from planetstack.config import Config +from ec2_observer.syncstep import SyncStep +from core.models.node import Node +from core.models.site import * +from ec2_observer.awslib import * +import pdb + +class SyncNodes(SyncStep): + provides=[Node] + requested_interval=0 + + def fetch_pending(self, deletion): + deployment = Deployment.objects.filter(Q(name="Amazon EC2"))[0] + current_site_deployments = SiteDeployments.objects.filter(Q(deployment=deployment)) + + zone_ret = aws_run('ec2 describe-availability-zones') + zones = zone_ret['AvailabilityZones'] + + # collect local nodes + instance_types = 'm1.small | m1.medium | m1.large | m1.xlarge | m3.medium | m3.large | m3.xlarge | m3.2xlarge'.split(' | ') + + all_new_nodes = [] + for sd in current_site_deployments: + s = sd.site + current_fqns = [n.name for n in s.nodes.all()] + all_fqns = ['.'.join([n,s.name]) for n in instance_types] + new_node_names = list(set(all_fqns) - set(current_fqns)) + + new_nodes = [] + for node_name in new_node_names: + node = Node(name=node_name, + site=s,deployment=deployment) + new_nodes.append(node) + + all_new_nodes.extend(new_nodes) + + return all_new_nodes + + + def sync_record(self, node): + node.save() + diff --git a/planetstack/ec2_observer/steps/sync_site_deployments.py b/planetstack/ec2_observer/steps/sync_site_deployments.py index a996c85..30fdf97 100644 --- a/planetstack/ec2_observer/steps/sync_site_deployments.py +++ b/planetstack/ec2_observer/steps/sync_site_deployments.py @@ -2,27 +2,46 @@ import os import base64 from django.db.models import F, Q from planetstack.config import Config -from observer.openstacksyncstep import OpenStackSyncStep +from ec2_observer.syncstep import SyncStep from core.models.site import * +from ec2_observer.awslib import * +import pdb -class SyncSiteDeployments(OpenStackSyncStep): - requested_interval=0 - provides=[Site, SiteDeployments] - - def fetch_pending(self): - return SiteDeployments.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None)) - - def sync_record(self, site_deployment): - if not site_deployment.tenant_id: - driver = self.driver.admin_driver(deployment=site_deployment.deployment.name) - tenant = driver.create_tenant(tenant_name=site_deployment.site.login_base, - description=site_deployment.site.name, - enabled=site_deployment.site.enabled) - site_deployment.tenant_id = tenant.id - site_deployment.save() - elif site_deployment.site.id and site_deployment.tenant_id: - driver = self.driver.admin_driver(deployment=site_deployment.name) - driver.update_tenant(site_deployment.tenant_id, - description=site_deployment.site.name, - enabled=site_deployment.site.enabled) - +class SyncSiteDeployments(SyncStep): + requested_interval=86400 + provides=[SiteDeployments] + + def fetch_pending(self, deletion): + zone_ret = aws_run('ec2 describe-availability-zones') + zones = zone_ret['AvailabilityZones'] + available_sites = [zone['ZoneName'] for zone in zones] + + current_sites = [] + for s in available_sites: + site = Site.objects.filter(Q(name=s)) + if (site): + current_sites.append(site[0]) + + # OK not to catch exception + # The syncstep should catch it + # At any rate, we should not run if there are no deployments + deployment = Deployment.objects.filter(Q(name="Amazon EC2"))[0] + current_site_deployments = SiteDeployments.objects.filter(Q(deployment=deployment)) + site_dict = {} + + for sd in current_site_deployments: + site_dict[sd.site]=sd + + updated_site_deployments = [] + for site in current_sites: + try: + site_record = site_dict[site] + except KeyError: + sd = SiteDeployments(site=site,deployment=deployment,tenant_id=base64.urlsafe_b64encode(os.urandom(12))) + updated_site_deployments.append(sd) + + return updated_site_deployments + + + def sync_record(self, site_deployment): + site_deployment.save() diff --git a/planetstack/ec2_observer/steps/sync_sites.py b/planetstack/ec2_observer/steps/sync_sites.py index 5771aef..7a5c0dc 100644 --- a/planetstack/ec2_observer/steps/sync_sites.py +++ b/planetstack/ec2_observer/steps/sync_sites.py @@ -2,20 +2,27 @@ import os import base64 from django.db.models import F, Q from planetstack.config import Config -from observer.syncstep import SyncStep -from core.models.site import Site +from ec2_observer.syncstep import SyncStep +from core.models.site import * from ec2_observer.awslib import * +import pdb class SyncSites(SyncStep): - provides=[Site] - requested_interval=3600 + provides=[Site] + requested_interval=3600 + + def fetch_pending(self, deletion): + + deployment = Deployment.objects.filter(Q(name="Amazon EC2"))[0] + current_site_deployments = SiteDeployments.objects.filter(Q(deployment=deployment)) + + zone_ret = aws_run('ec2 describe-availability-zones') + zones = zone_ret['AvailabilityZones'] - def fetch_pending(self): - current_sites = Site.objects.all() - zones = aws_run('ec2 describe-availability-zones') available_sites = [zone['ZoneName'] for zone in zones] + site_names = [sd.site.name for sd in current_site_deployments] - new_site_names = list(set(available_sites) - set(zones)) + new_site_names = list(set(available_sites) - set(site_names)) new_sites = [] for s in new_site_names: @@ -27,8 +34,8 @@ class SyncSites(SyncStep): abbreviated_name=s) new_sites.append(site) - return new_sites + return new_sites - def sync_record(self, site): - site.save() + def sync_record(self, site): + site.save() diff --git a/planetstack/ec2_observer/steps/sync_slivers.py b/planetstack/ec2_observer/steps/sync_slivers.py index b576bbc..2079859 100644 --- a/planetstack/ec2_observer/steps/sync_slivers.py +++ b/planetstack/ec2_observer/steps/sync_slivers.py @@ -1,91 +1,103 @@ import os +import json import base64 from django.db.models import F, Q from planetstack.config import Config -from observer.openstacksyncstep import OpenStackSyncStep +from ec2_observer.syncstep import SyncStep from core.models.sliver import Sliver from core.models.slice import SlicePrivilege, SliceDeployments from core.models.network import Network, NetworkSlice, NetworkDeployments from util.logger import Logger, logging +from ec2_observer.awslib import * +from core.models.site import * +from core.models.slice import * +import pdb logger = Logger(level=logging.INFO) -class SyncSlivers(OpenStackSyncStep): - provides=[Sliver] - requested_interval=0 +class SyncSlivers(SyncStep): + provides=[Sliver] + requested_interval=0 - def fetch_pending(self): - return Sliver.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None)) + def fetch_pending(self, deletion): + all_slivers = Sliver.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None)) + my_slivers = [] - def sync_record(self, sliver): - logger.info("sync'ing sliver:%s deployment:%s " % (sliver, sliver.node.deployment)) - metadata_update = {} - if ("numberCores" in sliver.changed_fields): - metadata_update["cpu_cores"] = str(sliver.numberCores) + for sliver in all_slivers: + sd = SliceDeployments.objects.filter(Q(slice=sliver.slice)) + if (sd): + if (sd.deployment.name=='Amazon EC2'): + my_slivers.append(sliver) + if (sliver.node.deployment.name=='Amazon EC2'): + my_slivers.append(sliver) + return my_slivers - for tag in sliver.slice.tags.all(): - if tag.name.startswith("sysctl-"): - metadata_update[tag.name] = tag.value + def sync_record(self, sliver): + logger.info("sync'ing sliver:%s deployment:%s " % (sliver, sliver.node.deployment)) - if not sliver.instance_id: - driver = self.driver.client_driver(caller=sliver.creator, tenant=sliver.slice.name, deployment=sliver.deploymentNetwork.name) - # public keys - slice_memberships = SlicePrivilege.objects.filter(slice=sliver.slice) - pubkeys = [sm.user.public_key for sm in slice_memberships if sm.user.public_key] - if sliver.creator.public_key: - pubkeys.append(sliver.creator.public_key) - if sliver.slice.creator.public_key: - pubkeys.append(sliver.slice.creator.public_key) - # netowrks - # include all networks available to the slice and/or associated network templates - nics = [] - networks = [ns.network for ns in NetworkSlice.objects.filter(slice=sliver.slice)] - network_deployments = NetworkDeployments.objects.filter(network__in=networks, - deployment=sliver.node.deployment) - # Gather private networks first. This includes networks with a template that has - # visibility = private and translation = none - for network_deployment in network_deployments: - if network_deployment.network.template.visibility == 'private' and \ - network_deployment.network.template.translation == 'none': - nics.append({'net-id': network_deployment.net_id}) - - # now include network template - network_templates = [network.template.sharedNetworkName for network in networks \ - if network.template.sharedNetworkName] - for net in driver.shell.quantum.list_networks()['networks']: - if net['name'] in network_templates: - nics.append({'net-id': net['id']}) + if not sliver.instance_id: + # public keys + slice_memberships = SlicePrivilege.objects.filter(slice=sliver.slice) + pubkeys = [sm.user.public_key for sm in slice_memberships if sm.user.public_key] - file("/tmp/scott-manager","a").write("slice: %s\nreq: %s\n" % (str(sliver.slice.name), str(nics))) - - # look up image id - deployment_driver = self.driver.admin_driver(deployment=sliver.deploymentNetwork.name) - image_id = None - images = deployment_driver.shell.glance.get_images() - for image in images: - if image['name'] == sliver.image.name: - image_id = image['id'] - - # look up key name at the deployment - # create/fetch keypair - keyname = None - if sliver.creator.public_key: - keyname = sliver.creator.email.lower().replace('@', 'AT').replace('.', '') +\ - sliver.slice.name - key_fields = {'name': keyname, - 'public_key': sliver.creator.public_key} - driver.create_keypair(**key_fields) - - instance = driver.spawn_instance(name=sliver.name, - key_name = keyname, - image_id = image_id, - hostname = sliver.node.name, - pubkeys = pubkeys, - nics = nics ) - sliver.instance_id = instance.id - sliver.instance_name = getattr(instance, 'OS-EXT-SRV-ATTR:instance_name') - sliver.save() + if sliver.creator.public_key: + pubkeys.append(sliver.creator.public_key) - if sliver.instance_id and metadata_update: - driver.update_instance_metadata(sliver.instance_id, metadata_update) + if sliver.slice.creator.public_key: + pubkeys.append(sliver.slice.creator.public_key) + + # netowrks + # include all networks available to the slice and/or associated network templates + #nics = [] + #networks = [ns.network for ns in NetworkSlice.objects.filter(slice=sliver.slice)] + #network_deployments = NetworkDeployments.objects.filter(network__in=networks, + #deployment=sliver.node.deployment) + # Gather private networks first. This includes networks with a template that has + # visibility = private and translation = none + #for network_deployment in network_deployments: + # if network_deployment.network.template.visibility == 'private' and \ + # network_deployment.network.template.translation == 'none': + # nics.append({'net-id': network_deployment.net_id}) + + # now include network template + #network_templates = [network.template.sharedNetworkName for network in networks \ + # if network.template.sharedNetworkName] + #for net in driver.shell.quantum.list_networks()['networks']: + # if net['name'] in network_templates: + # nics.append({'net-id': net['id']}) + # look up image id + + instance_type = sliver.node.name.rsplit('.',1)[0] + + # Bail out of we don't have a key + key_name = sliver.creator.email.lower().replace('@', 'AT').replace('.', '') + key_sig = aws_run('ec2 describe-key-pairs') + ec2_keys = key_sig['KeyPairs'] + key_found = False + for key in ec2_keys: + if (key['KeyName']==key_name): + key_found = True + break + + if (not key_found): + # set backend_status + raise Exception('Will not sync sliver without key') + + image_id = sliver.image.path + instance_sig = aws_run('ec2 run-instances --image-id %s --instance-type %s --count 1 --key-name %s --placement AvailabilityZone=%s'%(image_id,instance_type,key_name,sliver.node.site.name)) + sliver.instance_id = instance_sig['Instances'][0]['InstanceId'] + sliver.save() + state = instance_sig['Instances'][0]['State']['Code'] + if (state==16): + sliver.ip = instance_sig['Instances'][0]['PublicIpAddress'] + sliver.save() + else: + # This status message should go into backend_status + raise Exception('Waiting for instance to start') + else: + ret = aws_run('ec2 describe-instances --instance-ids %s'%sliver.instance_id) + state = ret['Reservations'][0]['Instances'][0]['State']['Code'] + if (state==16): + sliver.ip = ret['Reservations'][0]['Instances'][0]['PublicIpAddress'] + sliver.save() diff --git a/planetstack/ec2_observer/steps/sync_users.py b/planetstack/ec2_observer/steps/sync_users.py index 71f9c0f..d742186 100644 --- a/planetstack/ec2_observer/steps/sync_users.py +++ b/planetstack/ec2_observer/steps/sync_users.py @@ -1,20 +1,43 @@ import os import base64 -import hashlib +import random +import time +from datetime import datetime from django.db.models import F, Q from planetstack.config import Config -from observer.openstacksyncstep import OpenStackSyncStep -from core.models.user import User, UserDeployments +from ec2_observer.syncstep import SyncStep +from core.models.user import User +from core.models.site import * +from ec2_observer.awslib import * +import pdb -class SyncUsers(OpenStackSyncStep): - provides=[User] - requested_interval=0 +class SyncUsers(SyncStep): + provides=[User] + requested_interval=0 - def fetch_pending(self): - return User.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None)) + def fetch_pending(self, deletion): + users = User.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None)) + if (users): + key_sig = aws_run('ec2 describe-key-pairs') + ec2_keys = key_sig['KeyPairs'] + else: + ec2_keys = [] - def sync_record(self, user): - for user_deployment in UserDeployments.objects.filter(user=user): - # bump the 'updated' field so user account are updated across - # deployments. - user_deployment.save() + for user in users: + if (user.public_key): + key_name = user.email.lower().replace('@', 'AT').replace('.', '') + key_found = False + + for key in ec2_keys: + if (key['KeyName']==key_name): + key_found = True + break + + if (not key_found): + aws_run('ec2 import-key-pair --key-name %s --public-key-material "%s"'%(key_name, user.public_key)) + + return users + + def sync_record(self, node): + node.save() + -- 2.45.2