From: Thierry Parmentelat Date: Sun, 19 Feb 2012 08:29:59 +0000 (+0100) Subject: Merge branch 'master' into sqlalchemy X-Git-Tag: sfa-2.1-3~10 X-Git-Url: http://git.onelab.eu/?p=sfa.git;a=commitdiff_plain;h=78467ecd9b02ec60d9572178995ed347fc917f59;hp=470594b18884534906813e13b7150464824ace91 Merge branch 'master' into sqlalchemy Conflicts: config/default_config.xml sfa.spec Removed: sfa/server/sfa-clean-peer-records.py sfa/storage/record.py --- diff --git a/config/default_config.xml b/config/default_config.xml index 7eeb1c71..eeca137f 100644 --- a/config/default_config.xml +++ b/config/default_config.xml @@ -309,7 +309,30 @@ Thierry Parmentelat - + + + SFA Flash Policy + The settings that affect how SFA connects to + the Nova/EC2 API + + + Sfa nova user + novaadmin + Account/context to use when performing + administrative nova operations + + + Nova API url + 127.0.0.1 + The Nova/EC2 API url + + + Nova API Port + 8773 + The Nova/EC2 API port. + + + diff --git a/setup.py b/setup.py index e3bca577..4c20ceeb 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,6 @@ scripts = glob("sfa/clientbin/*.py") + \ 'sfa/importer/sfa-nuke.py', 'sfa/server/sfa-ca.py', 'sfa/server/sfa-start.py', - 'sfa/server/sfa-clean-peer-records.py', 'sfa/server/sfa_component_setup.py', 'sfatables/sfatables', 'keyconvert/keyconvert.py', diff --git a/sfa.spec b/sfa.spec index 678a1f75..4cbed111 100644 --- a/sfa.spec +++ b/sfa.spec @@ -168,7 +168,6 @@ rm -rf $RPM_BUILD_ROOT /etc/sfa/topology %{_bindir}/sfa-import.py* %{_bindir}/sfa-nuke.py* -%{_bindir}/sfa-clean-peer-records.py* %{_bindir}/gen-sfa-cm-config.py* %{_bindir}/sfa-ca.py* diff --git a/sfa/client/sfaadmin.py b/sfa/client/sfaadmin.py new file mode 100755 index 00000000..77f966f4 --- /dev/null +++ b/sfa/client/sfaadmin.py @@ -0,0 +1,191 @@ +#!/usr/bin/python +import sys +import copy +from sfa.generic import Generic +from optparse import OptionParser + +from sfa.util.xrn import Xrn +from sfa.storage.record import SfaRecord +from sfa.client.sfi import save_records_to_file + +def args(*args, **kwargs): + def _decorator(func): + func.__dict__.setdefault('options', []).insert(0, (args, kwargs)) + return func + return _decorator + +class Commands(object): + + def _get_commands(self): + available_methods = [] + for attrib in dir(self): + if callable(getattr(self, attrib)) and not attrib.startswith('_'): + available_methods.append(attrib) + return available_methods + +class RegistryCommands(Commands): + + def __init__(self, *args, **kwds): + self.api= Generic.the_flavour().make_api(interface='registry') + + def version(self): + pass + + @args('-x', '--xrn', dest='xrn', metavar='', help='object hrn/urn') + @args('-t', '--type', dest='type', metavar='', help='object type', default=None) + def list(self, xrn, type=None): + xrn = Xrn(xrn, type) + records = self.api.manager.List(self.api, xrn.get_hrn()) + for record in records: + if not type or record['type'] == type: + print "%s (%s)" % (record['hrn'], record['type']) + + + @args('-x', '--xrn', dest='xrn', metavar='', help='object hrn/urn') + @args('-t', '--type', dest='type', metavar='', help='object type', default=None) + @args('-o', '--outfile', dest='outfile', metavar='', help='save record to file') + @args('-f', '--format', dest='format', metavar='', type='choice', + choices=('text', 'xml', 'summary'), help='display record in different formats') + def show(self, xrn, type=None, format=None, outfile=None): + records = self.api.manager.Resolve(self.api, xrn, type, True) + for record in records: + sfa_record = SfaRecord(dict=record) + sfa_record.dump(format) + if outfile: + save_records_to_file(outfile, records) + + def register(self, record): + pass + + def update(self, record): + pass + + def remove(self, xrn): + pass + + def credential(self, xrn): + pass + + +class CerficiateCommands(Commands): + + def import_records(self, xrn): + pass + + def export(self, xrn): + pass + + + def display(self, xrn): + pass + def nuke(self): + pass + +class AggregateCommands(Commands): + + def __init__(self, *args, **kwds): + self.api= Generic.the_flavour().make_api(interface='aggregate') + + def version(self): + pass + + def slices(self): + pass + + def status(self, xrn): + pass + + def resources(self, xrn): + pass + + def create(self, xrn, rspec): + pass + + def delete(self, xrn): + pass + + def start(self, xrn): + pass + + def stop(self, xrn): + pass + + def reset(self, xrn): + pass + + def ticket(self): + pass + + +class SliceManagerCommands(AggregateCommands): + + def __init__(self, *args, **kwds): + self.api= Generic().make_api(interface='slicemgr') + + +CATEGORIES = {'registry': RegistryCommands, + 'aggregate': AggregateCommands, + 'slicemgr': SliceManagerCommands} + +def main(): + argv = copy.deepcopy(sys.argv) + script_name = argv.pop(0) + if len(argv) < 1: + print script_name + " category action []" + print "Available categories:" + for k in CATEGORIES: + print "\t%s" % k + sys.exit(2) + + category = argv.pop(0) + usage = "%%prog %s action [options]" % (category) + parser = OptionParser(usage=usage) + command_class = CATEGORIES[category] + command_instance = command_class() + actions = command_instance._get_commands() + if len(argv) < 1: + if hasattr(command_instance, '__call__'): + action = '' + command = command_instance.__call__ + else: + print script_name + " category action []" + print "Available actions for %s category:" % category + for k in actions: + print "\t%s" % k + sys.exit(2) + else: + action = argv.pop(0) + command = getattr(command_instance, action) + + options = getattr(command, 'options', []) + usage = "%%prog %s %s [options]" % (category, action) + parser = OptionParser(usage=usage) + for arg, kwd in options: + parser.add_option(*arg, **kwd) + (opts, cmd_args) = parser.parse_args(argv) + cmd_kwds = vars(opts) + + # dont overrride meth + for k, v in cmd_kwds.items(): + if v is None: + del cmd_kwds[k] + + try: + command(*cmd_args, **cmd_kwds) + sys.exit(0) + except TypeError: + print "Possible wrong number of arguments supplied" + print command.__doc__ + parser.print_help() + #raise + except Exception: + print "Command failed, please check log for more info" + raise + + +if __name__ == '__main__': + main() + + + + diff --git a/sfa/managers/registry_manager.py b/sfa/managers/registry_manager.py index 2de48751..331a0f94 100644 --- a/sfa/managers/registry_manager.py +++ b/sfa/managers/registry_manager.py @@ -101,10 +101,10 @@ class RegistryManager: def Resolve(self, api, xrns, type=None, full=True): if not isinstance(xrns, types.ListType): - xrns = [xrns] # try to infer type if not set and we get a single input if not type: type = Xrn(xrns).get_type() + xrns = [xrns] hrns = [urn_to_hrn(xrn)[0] for xrn in xrns] # load all known registry names into a prefix tree and attempt to find diff --git a/sfa/openstack/euca_shell.py b/sfa/openstack/euca_shell.py new file mode 100644 index 00000000..80f9f524 --- /dev/null +++ b/sfa/openstack/euca_shell.py @@ -0,0 +1,59 @@ +try: + import boto + from boto.ec2.regioninfo import RegionInfo + from boto.exception import EC2ResponseError + has_boto=True +except: + has_boto=False + +from sfa.util.sfalogging import logger +from sfa.openstack.nova_shell import NovaShell +from sfa.util.config import Config + +class EucaShell: + """ + A xmlrpc connection to the euca api. + """ + + def __init__(self, config): + self.config = Config + + def get_euca_connection(self): + if not has_boto: + logger.info('Unable to access EC2 API - boto library not found.') + return None + nova = NovaShell(self.config) + admin_user = nova.auth_manager.get_user(self.config.SFA_NOVA_USER) + access_key = admin_user.access + secret_key = admin_user.secret + url = self.config.SFA_NOVA_API_URL + path = "/" + euca_port = self.config.SFA_NOVA_API_PORT + use_ssl = False + + # Split the url into parts + if url.find('https://') >= 0: + use_ssl = True + url = url.replace('https://', '') + elif url.find('http://') >= 0: + use_ssl = False + url = url.replace('http://', '') + (host, parts) = url.split(':') + if len(parts) > 1: + parts = parts.split('/') + port = int(parts[0]) + parts = parts[1:] + path = '/'.join(parts) + + return boto.connect_ec2(aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + is_secure=use_ssl, + region=RegionInfo(None, 'eucalyptus', host), + port=port, + path=path) + + def __getattr__(self, name): + def func(*args, **kwds): + conn = self.get_euca_connection() + return getattr(conn, name)(*args, **kwds) + return func diff --git a/sfa/openstack/nova_driver.py b/sfa/openstack/nova_driver.py index 404021be..1d3eff38 100644 --- a/sfa/openstack/nova_driver.py +++ b/sfa/openstack/nova_driver.py @@ -18,6 +18,7 @@ from sfa.rspecs.rspec import RSpec # the driver interface, mostly provides default behaviours from sfa.managers.driver import Driver from sfa.openstack.nova_shell import NovaShell +from sfa.openstack.euca_shell import EucaShell from sfa.openstack.osaggregate import OSAggregate from sfa.plc.plslices import PlSlices from sfa.util.osxrn import OSXrn @@ -43,6 +44,7 @@ class NovaDriver (Driver): def __init__ (self, config): Driver.__init__ (self, config) self.shell = NovaShell (config) + self.euca_shell = EucaShell(config) self.cache=None if config.SFA_AGGREGATE_CACHING: if NovaDriver.cache is None: @@ -244,49 +246,37 @@ class NovaDriver (Driver): def sliver_status (self, slice_urn, slice_hrn): # find out where this slice is currently running - slicename = hrn_to_pl_slicename(slice_hrn) - - slices = self.shell.GetSlices([slicename], ['slice_id', 'node_ids','person_ids','name','expires']) - if len(slices) == 0: - raise SliverDoesNotExist("%s (used %s as slicename internally)" % (slice_hrn, slicename)) - slice = slices[0] - - # report about the local nodes only - nodes = self.shell.GetNodes({'node_id':slice['node_ids'],'peer_id':None}, - ['node_id', 'hostname', 'site_id', 'boot_state', 'last_contact']) - - if len(nodes) == 0: + project_name = Xrn(slice_urn).get_leaf() + project = self.shell.auth_manager.get_project(project_name) + instances = self.shell.db.instance_get_all_by_project(project_name) + if len(instances) == 0: raise SliverDoesNotExist("You have not allocated any slivers here") - - site_ids = [node['site_id'] for node in nodes] - + result = {} top_level_status = 'unknown' - if nodes: + if instances: top_level_status = 'ready' result['geni_urn'] = slice_urn - result['pl_login'] = slice['name'] - result['pl_expires'] = datetime_to_string(utcparse(slice['expires'])) + result['plos_login'] = 'root' + result['plos_expires'] = None resources = [] - for node in nodes: + for instance in instances: res = {} - res['pl_hostname'] = node['hostname'] - res['pl_boot_state'] = node['boot_state'] - res['pl_last_contact'] = node['last_contact'] - if node['last_contact'] is not None: - - res['pl_last_contact'] = datetime_to_string(utcparse(node['last_contact'])) - sliver_id = urn_to_sliver_id(slice_urn, slice['slice_id'], node['node_id']) + # instances are accessed by ip, not hostname. We need to report the ip + # somewhere so users know where to ssh to. + res['plos_hostname'] = instance.hostname + res['plos_created_at'] = datetime_to_string(utcparse(instance.created_at)) + res['plos_boot_state'] = instance.vm_state + res['plos_sliver_type'] = instance.instance_type.name + sliver_id = Xrn(slice_urn).get_sliver_id(instance.project_id, \ + instance.hostname, instance.id) res['geni_urn'] = sliver_id - if node['boot_state'] == 'boot': - res['geni_status'] = 'ready' + + if instance.vm_state == 'running': + res['boot_state'] = 'ready'; else: - res['geni_status'] = 'failed' - top_level_status = 'failed' - - res['geni_error'] = '' - + res['boot_state'] = 'unknown' resources.append(res) result['geni_status'] = top_level_status @@ -301,13 +291,19 @@ class NovaDriver (Driver): # parse rspec rspec = RSpec(rspec_string) requested_attributes = rspec.version.get_slice_attributes() + pubkeys = [] + for user in users: + pubkeys.extend(user['keys']) + # assume that there is a key whos nane matches the caller's username. + project_key = Xrn(users[0]['urn']).get_leaf() + # ensure slice record exists - slice = aggregate.verify_slice(slicename, users, options=options) + aggregate.create_project(slicename, users, options=options) # ensure person records exists - persons = aggregate.verify_slice_users(slicename, users, options=options) + aggregate.create_project_users(slicename, users, options=options) # add/remove slice from nodes - slices.verify_instances(slicename, rspec) + aggregate.run_instances(slicename, rspec, project_key, pubkeys) return aggregate.get_rspec(slice_xrn=slice_urn, version=rspec.version) @@ -316,8 +312,6 @@ class NovaDriver (Driver): slice = self.shell.project_get(name) if not slice: return 1 - - self.shell.DeleteSliceFromNodes(slicename, slice['node_ids']) instances = self.shell.db.instance_get_all_by_project(name) for instance in instances: self.shell.db.instance_destroy(instance.instance_id) diff --git a/sfa/openstack/nova_shell.py b/sfa/openstack/nova_shell.py index 214bb7d1..9179faa1 100644 --- a/sfa/openstack/nova_shell.py +++ b/sfa/openstack/nova_shell.py @@ -33,8 +33,8 @@ class InjectContext: class NovaShell: """ - A simple xmlrpc shell to a myplc instance - This class can receive all Openstack calls to the underlying testbed + A simple native shell to a nova backend. + This class can receive all nova calls to the underlying testbed """ # dont care about limiting calls yet diff --git a/sfa/openstack/osaggregate.py b/sfa/openstack/osaggregate.py index 60e3d56d..e7486e82 100644 --- a/sfa/openstack/osaggregate.py +++ b/sfa/openstack/osaggregate.py @@ -1,4 +1,5 @@ - +from nova.exception import ImageNotFound +from nova.api.ec2.cloud import CloudController from sfa.util.faults import SfaAPIError from sfa.rspecs.rspec import RSpec from sfa.rspecs.elements.hardware_type import HardwareType @@ -11,40 +12,87 @@ from sfa.util.xrn import Xrn from sfa.util.osxrn import OSXrn from sfa.rspecs.version_manager import VersionManager + +def disk_image_to_rspec_object(image): + img = DiskImage() + img['name'] = image['name'] + img['description'] = image['name'] + img['os'] = image['name'] + img['version'] = image['name'] + return img + + +def instance_to_sliver(instance, slice_xrn=None): + # should include? + # * instance.image_ref + # * instance.kernel_id + # * instance.ramdisk_id + import nova.db.sqlalchemy.models + name=None + type=None + sliver_id = None + if isinstance(instance, dict): + # this is an isntance type dict + name = instance['name'] + type = instance['name'] + elif isinstance(instance, nova.db.sqlalchemy.models.Instance): + # this is an object that describes a running instance + name = instance.display_name + type = instance.instance_type.name + else: + raise SfaAPIError("instnace must be an instance_type dict or" + \ + " a nova.db.sqlalchemy.models.Instance object") + if slice_xrn: + xrn = Xrn(slice_xrn, 'slice') + sliver_id = xrn.get_sliver_id(instance.project_id, instance.hostname, instance.id) + + sliver = Sliver({'slice_id': sliver_id, + 'name': name, + 'type': 'plos-' + type, + 'tags': []}) + return sliver + + class OSAggregate: def __init__(self, driver): self.driver = driver - def instance_to_sliver(self, instance, slice_xrn=None): - # should include? - # * instance.image_ref - # * instance.kernel_id - # * instance.ramdisk_id - import nova.db.sqlalchemy.models - name=None - type=None - sliver_id = None - if isinstance(instance, dict): - # this is an isntance type dict - name = instance['name'] - type = instance['name'] - elif isinstance(instance, nova.db.sqlalchemy.models.Instance): - # this is an object that describes a running instance - name = instance.display_name - type = instance.instance_type.name - else: - raise SfaAPIError("instnace must be an instance_type dict or" + \ - " a nova.db.sqlalchemy.models.Instance object") - if slice_xrn: - xrn = Xrn(slice_xrn, 'slice') - sliver_id = xrn.get_sliver_id(instance.project_id, instance.hostname, instance.id) - - sliver = Sliver({'slice_id': sliver_id, - 'name': name, - 'type': 'plos-' + type, - 'tags': []}) - return sliver + def get_machine_image_details(self, image): + """ + Returns a dict that contains the ami, aki and ari details for the specified + ami image. + """ + disk_image = {} + if image['container_format'] == 'ami': + disk_image['ami'] = image + disk_image['aki'] = self.driver.shell.image_manager.show(image['kernel_id']) + disk_image['ari'] = self.driver.shell.image_manager.show(image['ramdisk_id']) + return disk_image + + def get_disk_image(self, id=None, name=None): + """ + Look up a image bundle using the specifeid id or name + """ + disk_image = None + try: + if id: + image = self.driver.shell.image_manager.show(image_id) + elif name: + image = self.driver.shell.image_manager.show_by_name(image_name) + if image['container_format'] == 'ami': + disk_image = self.get_machine_image_details(image) + except ImageNotFound: + pass + return disk_image + + def get_available_disk_images(self): + # get image records + disk_images = [] + for image in self.driver.shell.image_manager.detail(): + if image['container_format'] == 'ami': + disk_images.append(self.get_machine_image_details(image)) + return disk_images def get_rspec(self, slice_xrn=None, version=None, options={}): version_manager = VersionManager() @@ -69,7 +117,9 @@ class OSAggregate: rspec_node['component_id'] = xrn.urn rspec_node['component_name'] = xrn.name rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn() - sliver = self.instance_to_sliver(instance) + sliver = instance_to_sliver(instance) + disk_image = self.get_disk_image(instance.image_ref) + sliver['disk_images'] = [disk_image_to_rspec_object(disk_image)] rspec_node['slivers'] = [sliver] rspec_nodes.append(rspec_node) return rspec_nodes @@ -85,17 +135,9 @@ class OSAggregate: # available sliver/instance/vm types instances = self.driver.shell.db.instance_type_get_all().values() # available images - images = self.driver.shell.image_manager.detail() - disk_images = [] - for image in images: - if image['container_format'] == 'ami': - img = DiskImage() - img['name'] = image['name'] - img['description'] = image['name'] - img['os'] = image['name'] - img['version'] = image['name'] - disk_images.append(img) - + disk_images = self.get_available_disk_images() + disk_image_objects = [disk_image_to_rspec_object(image) \ + for image in disk_image] rspec_nodes = [] for zone in zones: rspec_node = Node() @@ -108,8 +150,8 @@ class OSAggregate: HardwareType({'name': 'pc'})] slivers = [] for instance in instances: - sliver = self.instance_to_sliver(instance) - sliver['disk_images'] = disk_images + sliver = instance_to_sliver(instance) + sliver['disk_images'] = disk_image_objects slivers.append(sliver) rspec_node['slivers'] = slivers @@ -118,7 +160,7 @@ class OSAggregate: return rspec_nodes - def verify_slice(self, slicename, users, options={}): + def create_project(self, slicename, users, options={}): """ Create the slice if it doesn't alredy exist """ @@ -132,7 +174,7 @@ class OSAggregate: proj_manager = usernames[0] self.driver.shell.auth_manager.create_project(slicename, proj_manager) - def verify_slice_users(self, slicename, users, options={}): + def create_project_users(self, slicename, users, options={}): """ Add requested users to the specified slice. """ @@ -148,6 +190,7 @@ class OSAggregate: except nova.exception.UserNotFound: self.driver.shell.auth_manager.create_user(username) self.verify_user_keys(username, user['keys'], options) + def verify_user_keys(self, username, keys, options={}): """ @@ -158,7 +201,7 @@ class OSAggregate: existing_pub_keys = [key.public_key for key in existing_keys] removed_pub_keys = set(existing_pub_keys).difference(keys) added_pub_keys = set(keys).difference(existing_pub_keys) - + pubkeys = [] # add new keys for public_key in added_pub_keys: key = {} @@ -172,14 +215,57 @@ class OSAggregate: for key in existing_keys: if key.public_key in removed_pub_keys: self.driver.shell.db.key_pair_destroy(username, key.name) - - def verify_instances(self, slicename, rspec): - rsepc = RSpec(rspec) - nodes = rspec.version.get_nodes_with_slivers() - old_instances = self.driver.shell.db.instance_get_all_by_project(name) - for node in nodes: - for slivers in node.get('slivers', []): - pass - # get instance type - # get image - # start instance + + def reserve_instance(self, image_id, kernel_id, ramdisk_id, \ + instance_type, key_name, user_data): + conn = self.driver.euca_shell + logger.info('Reserving an instance: image: %s, kernel: ' \ + '%s, ramdisk: %s, type: %s, key: %s' % \ + (image_id, kernel_id, ramdisk_id, + instance_type, key_name)) + try: + reservation = conn.run_instances(image_id=image_id, + kernel_id=kernel_id, + ramdisk_id=ramdisk_id, + instance_type=instance_type, + key_name=key_name, + user_data = user_data) + except EC2ResponseError, ec2RespError: + logger.log_exc(ec2RespError) + + def run_instances(self, slicename, rspec, keyname, pubkeys): + """ + Create the instances thats requested in the rspec + """ + # the default image to use for instnaces that dont + # explicitly request an image. + # Just choose the first available image for now. + available_images = self.get_available_disk_images() + default_image = self.get_disk_images()[0] + default_ami_id = CloudController.image_ec2_id(default_image['ami']['id']) + default_aki_id = CloudController.image_ec2_id(default_image['aki']['id']) + default_ari_id = CloudController.image_ec2_id(default_image['ari']['id']) + + # get requested slivers + rspec = RSpec(rspec) + requested_instances = defaultdict(list) + # iterate over clouds/zones/nodes + for node in rspec.version.get_nodes_with_slivers(): + instance_types = node.get('slivers', []) + if isinstance(instance_types, list): + # iterate over sliver/instance types + for instance_type in instance_types: + ami_id = default_ami_id + aki_id = default_aki_id + ari_id = default_ari_id + req_image = instance_type.get('disk_images') + if req_image and isinstance(req_image, list): + req_image_name = req_image[0]['name'] + disk_image = self.get_disk_image(name=req_image_name) + if disk_image: + ami_id = CloudController.image_ec2_id(disk_image['ami']['id']) + aki_id = CloudController.image_ec2_id(disk_image['aki']['id']) + ari_id = CloudController.image_ec2_id(disk_image['ari']['id']) + # start the instance + self.reserve_instance(ami_id, aki_id, ari_id, \ + instance_type['name'], keyname, pubkeys) diff --git a/sfa/plc/plaggregate.py b/sfa/plc/plaggregate.py index 1cace7a9..55e6291e 100644 --- a/sfa/plc/plaggregate.py +++ b/sfa/plc/plaggregate.py @@ -136,6 +136,11 @@ class PlAggregate: return (slice, slivers) def get_nodes_and_links(self, slice=None,slivers=[], options={}): + # if we are dealing with a slice that has no node just return + # and empty list + if slice is not None and not slice['node_ids']: + return ([],[]) + filter = {} tags_filter = {} if slice and 'node_ids' in slice and slice['node_ids']: diff --git a/sfa/server/sfa-clean-peer-records.py b/sfa/server/sfa-clean-peer-records.py deleted file mode 100644 index 795c747f..00000000 --- a/sfa/server/sfa-clean-peer-records.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/python - -import sys -import os -import traceback -import socket - -from sfa.util.prefixTree import prefixTree -from sfa.util.config import Config - -from sfa.trust.certificate import Keypair -from sfa.trust.hierarchy import Hierarchy -from sfa.server.registry import Registries - -from sfa.storage.alchemy import dbsession -from sfa.storage.model import RegRecord - -from sfa.client.sfaserverproxy import SfaServerProxy - -from sfa.generic import Generic - -def main(): - config = Config() - if not config.SFA_REGISTRY_ENABLED: - sys.exit(0) - - # Get the path to the sfa server key/cert files from - # the sfa hierarchy object - sfa_hierarchy = Hierarchy() - auth_info = sfa_hierarchy.get_interface_auth_info() - key_file = auth_info.get_privkey_filename() - cert_file = auth_info.get_gid_filename() - key = Keypair(filename=key_file) - - # get a connection to our local sfa registry - # and a valid credential - authority = config.SFA_INTERFACE_HRN - url = 'http://%s:%s/' %(config.SFA_REGISTRY_HOST, config.SFA_REGISTRY_PORT) - registry = SfaServerProxy(url, key_file, cert_file) - sfa_api = Generic.the_flavour() - credential = sfa_api.getCredential() - - # get peer registries - registries = Registries(sfa_api) - tree = prefixTree() - tree.load(registries.keys()) - - # get local peer records - peer_records=dbsession.query(RegRecord).filter (RegRecord.peer_authority != None).all() - found_records = [] - hrn_dict = {} - for record in peer_records: - registry_hrn = tree.best_match(record.hrn) - if registry_hrn not in hrn_dict: - hrn_dict[registry_hrn] = [] - hrn_dict[registry_hrn].append(record.hrn) - - # attempt to resolve the record at the authoritative interface - for registry_hrn in hrn_dict: - if registry_hrn in registries: - records = [] - target_hrns = hrn_dict[registry_hrn] - try: - records = registries[registry_hrn].Resolve(target_hrns, credential) - found_records.extend([record['hrn'] for record in records]) - except ServerException: - # an exception will be thrown if the record doenst exist - # if so remove the record from the local registry - continue - except: - # this deosnt necessarily mean the records dont exist - # lets give them the benefit of the doubt here (for now) - found_records.extend(target_hrns) - traceback.print_exc() - - # remove what wasnt found - for peer_record in peer_records: - if peer_record.hrn not in found_records: - registries[sfa_api.hrn].Remove(peer_record.hrn, credential, peer_record.type) - -if __name__ == '__main__': - main() diff --git a/sfa/trust/gid.py b/sfa/trust/gid.py index 656de4be..470757bd 100644 --- a/sfa/trust/gid.py +++ b/sfa/trust/gid.py @@ -1,240 +1,260 @@ -#---------------------------------------------------------------------- -# Copyright (c) 2008 Board of Trustees, Princeton University -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and/or hardware specification (the "Work") to -# deal in the Work without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Work, and to permit persons to whom the Work -# is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Work. -# -# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS -# IN THE WORK. -#---------------------------------------------------------------------- -## -# Implements SFA GID. GIDs are based on certificates, and the GID class is a -# descendant of the certificate class. -## - -import xmlrpclib -import uuid - -from sfa.trust.certificate import Certificate - -from sfa.util.faults import GidInvalidParentHrn, GidParentHrn -from sfa.util.sfalogging import logger -from sfa.util.xrn import hrn_to_urn, urn_to_hrn, hrn_authfor_hrn - -## -# Create a new uuid. Returns the UUID as a string. - -def create_uuid(): - return str(uuid.uuid4().int) - -## -# GID is a tuple: -# (uuid, urn, public_key) -# -# UUID is a unique identifier and is created by the python uuid module -# (or the utility function create_uuid() in gid.py). -# -# HRN is a human readable name. It is a dotted form similar to a backward domain -# name. For example, planetlab.us.arizona.bakers. -# -# URN is a human readable identifier of form: -# "urn:publicid:IDN+toplevelauthority[:sub-auth.]*[\res. type]\ +object name" -# For example, urn:publicid:IDN+planetlab:us:arizona+user+bakers -# -# PUBLIC_KEY is the public key of the principal identified by the UUID/HRN. -# It is a Keypair object as defined in the cert.py module. -# -# It is expected that there is a one-to-one pairing between UUIDs and HRN, -# but it is uncertain how this would be inforced or if it needs to be enforced. -# -# These fields are encoded using xmlrpc into the subjectAltName field of the -# x509 certificate. Note: Call encode() once the fields have been filled in -# to perform this encoding. - - -class GID(Certificate): - uuid = None - hrn = None - urn = None - - ## - # Create a new GID object - # - # @param create If true, create the X509 certificate - # @param subject If subject!=None, create the X509 cert and set the subject name - # @param string If string!=None, load the GID from a string - # @param filename If filename!=None, load the GID from a file - # @param lifeDays life of GID in days - default is 1825==5 years - - def __init__(self, create=False, subject=None, string=None, filename=None, uuid=None, hrn=None, urn=None, lifeDays=1825): - - Certificate.__init__(self, lifeDays, create, subject, string, filename) - if subject: - logger.debug("Creating GID for subject: %s" % subject) - if uuid: - self.uuid = int(uuid) - if hrn: - self.hrn = hrn - self.urn = hrn_to_urn(hrn, 'unknown') - if urn: - self.urn = urn - self.hrn, type = urn_to_hrn(urn) - - def set_uuid(self, uuid): - if isinstance(uuid, str): - self.uuid = int(uuid) - else: - self.uuid = uuid - - def get_uuid(self): - if not self.uuid: - self.decode() - return self.uuid - - def set_hrn(self, hrn): - self.hrn = hrn - - def get_hrn(self): - if not self.hrn: - self.decode() - return self.hrn - - def set_urn(self, urn): - self.urn = urn - self.hrn, type = urn_to_hrn(urn) - - def get_urn(self): - if not self.urn: - self.decode() - return self.urn - - def get_type(self): - if not self.urn: - self.decode() - _, t = urn_to_hrn(self.urn) - return t - - ## - # Encode the GID fields and package them into the subject-alt-name field - # of the X509 certificate. This must be called prior to signing the - # certificate. It may only be called once per certificate. - - def encode(self): - if self.urn: - urn = self.urn - else: - urn = hrn_to_urn(self.hrn, None) - - str = "URI:" + urn - - if self.uuid: - str += ", " + "URI:" + uuid.UUID(int=self.uuid).urn - - self.set_data(str, 'subjectAltName') - - - - - ## - # Decode the subject-alt-name field of the X509 certificate into the - # fields of the GID. This is automatically called by the various get_*() - # functions in this class. - - def decode(self): - data = self.get_data('subjectAltName') - dict = {} - if data: - if data.lower().startswith('uri:http://'): - dict = xmlrpclib.loads(data[11:])[0][0] - else: - spl = data.split(', ') - for val in spl: - if val.lower().startswith('uri:urn:uuid:'): - dict['uuid'] = uuid.UUID(val[4:]).int - elif val.lower().startswith('uri:urn:publicid:idn+'): - dict['urn'] = val[4:] - - self.uuid = dict.get("uuid", None) - self.urn = dict.get("urn", None) - self.hrn = dict.get("hrn", None) - if self.urn: - self.hrn = urn_to_hrn(self.urn)[0] - - ## - # Dump the credential to stdout. - # - # @param indent specifies a number of spaces to indent the output - # @param dump_parents If true, also dump the parents of the GID - - def dump(self, *args, **kwargs): - print self.dump_string(*args,**kwargs) - - def dump_string(self, indent=0, dump_parents=False): - result=" "*(indent-2) + "GID\n" - result += " "*indent + "hrn:" + str(self.get_hrn()) +"\n" - result += " "*indent + "urn:" + str(self.get_urn()) +"\n" - result += " "*indent + "uuid:" + str(self.get_uuid()) + "\n" - filename=self.get_filename() - if filename: result += "Filename %s\n"%filename - - if self.parent and dump_parents: - result += " "*indent + "parent:\n" - result += self.parent.dump_string(indent+4, dump_parents) - return result - - ## - # Verify the chain of authenticity of the GID. First perform the checks - # of the certificate class (verifying that each parent signs the child, - # etc). In addition, GIDs also confirm that the parent's HRN is a prefix - # of the child's HRN, and the parent is of type 'authority'. - # - # Verifying these prefixes prevents a rogue authority from signing a GID - # for a principal that is not a member of that authority. For example, - # planetlab.us.arizona cannot sign a GID for planetlab.us.princeton.foo. - - def verify_chain(self, trusted_certs = None): - # do the normal certificate verification stuff - trusted_root = Certificate.verify_chain(self, trusted_certs) - - if self.parent: - # make sure the parent's hrn is a prefix of the child's hrn - if not hrn_authfor_hrn(self.parent.get_hrn(), self.get_hrn()): - raise GidParentHrn("This cert HRN %s isn't in the namespace for parent HRN %s" % (self.get_hrn(), self.parent.get_hrn())) - - # Parent must also be an authority (of some type) to sign a GID - # There are multiple types of authority - accept them all here - if not self.parent.get_type().find('authority') == 0: - raise GidInvalidParentHrn("This cert %s's parent %s is not an authority (is a %s)" % (self.get_hrn(), self.parent.get_hrn(), self.parent.get_type())) - - # Then recurse up the chain - ensure the parent is a trusted - # root or is in the namespace of a trusted root - self.parent.verify_chain(trusted_certs) - else: - # make sure that the trusted root's hrn is a prefix of the child's - trusted_gid = GID(string=trusted_root.save_to_string()) - trusted_type = trusted_gid.get_type() - trusted_hrn = trusted_gid.get_hrn() - #if trusted_type == 'authority': - # trusted_hrn = trusted_hrn[:trusted_hrn.rindex('.')] - cur_hrn = self.get_hrn() - if not hrn_authfor_hrn(trusted_hrn, cur_hrn): - raise GidParentHrn("Trusted root with HRN %s isn't a namespace authority for this cert %s" % (trusted_hrn, cur_hrn)) - - # There are multiple types of authority - accept them all here - if not trusted_type.find('authority') == 0: - raise GidInvalidParentHrn("This cert %s's trusted root signer %s is not an authority (is a %s)" % (self.get_hrn(), trusted_hrn, trusted_type)) - - return +#---------------------------------------------------------------------- +# Copyright (c) 2008 Board of Trustees, Princeton University +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and/or hardware specification (the "Work") to +# deal in the Work without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Work, and to permit persons to whom the Work +# is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Work. +# +# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS +# IN THE WORK. +#---------------------------------------------------------------------- +## +# Implements SFA GID. GIDs are based on certificates, and the GID class is a +# descendant of the certificate class. +## + +import xmlrpclib +import uuid + +from sfa.trust.certificate import Certificate + +from sfa.util.faults import GidInvalidParentHrn, GidParentHrn +from sfa.util.sfalogging import logger +from sfa.util.xrn import hrn_to_urn, urn_to_hrn, hrn_authfor_hrn + +## +# Create a new uuid. Returns the UUID as a string. + +def create_uuid(): + return str(uuid.uuid4().int) + +## +# GID is a tuple: +# (uuid, urn, public_key) +# +# UUID is a unique identifier and is created by the python uuid module +# (or the utility function create_uuid() in gid.py). +# +# HRN is a human readable name. It is a dotted form similar to a backward domain +# name. For example, planetlab.us.arizona.bakers. +# +# URN is a human readable identifier of form: +# "urn:publicid:IDN+toplevelauthority[:sub-auth.]*[\res. type]\ +object name" +# For example, urn:publicid:IDN+planetlab:us:arizona+user+bakers +# +# PUBLIC_KEY is the public key of the principal identified by the UUID/HRN. +# It is a Keypair object as defined in the cert.py module. +# +# It is expected that there is a one-to-one pairing between UUIDs and HRN, +# but it is uncertain how this would be inforced or if it needs to be enforced. +# +# These fields are encoded using xmlrpc into the subjectAltName field of the +# x509 certificate. Note: Call encode() once the fields have been filled in +# to perform this encoding. + + +class GID(Certificate): + uuid = None + hrn = None + urn = None + email = None # for adding to the SubjectAltName + + ## + # Create a new GID object + # + # @param create If true, create the X509 certificate + # @param subject If subject!=None, create the X509 cert and set the subject name + # @param string If string!=None, load the GID from a string + # @param filename If filename!=None, load the GID from a file + # @param lifeDays life of GID in days - default is 1825==5 years + + def __init__(self, create=False, subject=None, string=None, filename=None, uuid=None, hrn=None, urn=None, lifeDays=1825): + + Certificate.__init__(self, lifeDays, create, subject, string, filename) + if subject: + logger.debug("Creating GID for subject: %s" % subject) + if uuid: + self.uuid = int(uuid) + if hrn: + self.hrn = hrn + self.urn = hrn_to_urn(hrn, 'unknown') + if urn: + self.urn = urn + self.hrn, type = urn_to_hrn(urn) + + def set_uuid(self, uuid): + if isinstance(uuid, str): + self.uuid = int(uuid) + else: + self.uuid = uuid + + def get_uuid(self): + if not self.uuid: + self.decode() + return self.uuid + + def set_hrn(self, hrn): + self.hrn = hrn + + def get_hrn(self): + if not self.hrn: + self.decode() + return self.hrn + + def set_urn(self, urn): + self.urn = urn + self.hrn, type = urn_to_hrn(urn) + + def get_urn(self): + if not self.urn: + self.decode() + return self.urn + + # Will be stuffed into subjectAltName + def set_email(self, email): + self.email = email + + def get_email(self): + if not self.email: + self.decode() + return self.email + + def get_type(self): + if not self.urn: + self.decode() + _, t = urn_to_hrn(self.urn) + return t + + ## + # Encode the GID fields and package them into the subject-alt-name field + # of the X509 certificate. This must be called prior to signing the + # certificate. It may only be called once per certificate. + + def encode(self): + if self.urn: + urn = self.urn + else: + urn = hrn_to_urn(self.hrn, None) + + str = "URI:" + urn + + if self.uuid: + str += ", " + "URI:" + uuid.UUID(int=self.uuid).urn + + if self.email: + str += ", " + "email:" + self.email + + self.set_data(str, 'subjectAltName') + + + + + ## + # Decode the subject-alt-name field of the X509 certificate into the + # fields of the GID. This is automatically called by the various get_*() + # functions in this class. + + def decode(self): + data = self.get_data('subjectAltName') + dict = {} + if data: + if data.lower().startswith('uri:http://'): + dict = xmlrpclib.loads(data[11:])[0][0] + else: + spl = data.split(', ') + for val in spl: + if val.lower().startswith('uri:urn:uuid:'): + dict['uuid'] = uuid.UUID(val[4:]).int + elif val.lower().startswith('uri:urn:publicid:idn+'): + dict['urn'] = val[4:] + elif val.lower().startswith('email:'): + # FIXME: Ensure there isn't cruft in that address... + # EG look for email:copy,.... + dict['email'] = val[6:] + + self.uuid = dict.get("uuid", None) + self.urn = dict.get("urn", None) + self.hrn = dict.get("hrn", None) + self.email = dict.get("email", None) + if self.urn: + self.hrn = urn_to_hrn(self.urn)[0] + + ## + # Dump the credential to stdout. + # + # @param indent specifies a number of spaces to indent the output + # @param dump_parents If true, also dump the parents of the GID + + def dump(self, *args, **kwargs): + print self.dump_string(*args,**kwargs) + + def dump_string(self, indent=0, dump_parents=False): + result=" "*(indent-2) + "GID\n" + result += " "*indent + "hrn:" + str(self.get_hrn()) +"\n" + result += " "*indent + "urn:" + str(self.get_urn()) +"\n" + result += " "*indent + "uuid:" + str(self.get_uuid()) + "\n" + if self.get_email() is not None: + result += " "*indent + "email:" + str(self.get_email()) + "\n" + filename=self.get_filename() + if filename: result += "Filename %s\n"%filename + + if self.parent and dump_parents: + result += " "*indent + "parent:\n" + result += self.parent.dump_string(indent+4, dump_parents) + return result + + ## + # Verify the chain of authenticity of the GID. First perform the checks + # of the certificate class (verifying that each parent signs the child, + # etc). In addition, GIDs also confirm that the parent's HRN is a prefix + # of the child's HRN, and the parent is of type 'authority'. + # + # Verifying these prefixes prevents a rogue authority from signing a GID + # for a principal that is not a member of that authority. For example, + # planetlab.us.arizona cannot sign a GID for planetlab.us.princeton.foo. + + def verify_chain(self, trusted_certs = None): + # do the normal certificate verification stuff + trusted_root = Certificate.verify_chain(self, trusted_certs) + + if self.parent: + # make sure the parent's hrn is a prefix of the child's hrn + if not hrn_authfor_hrn(self.parent.get_hrn(), self.get_hrn()): + raise GidParentHrn("This cert HRN %s isn't in the namespace for parent HRN %s" % (self.get_hrn(), self.parent.get_hrn())) + + # Parent must also be an authority (of some type) to sign a GID + # There are multiple types of authority - accept them all here + if not self.parent.get_type().find('authority') == 0: + raise GidInvalidParentHrn("This cert %s's parent %s is not an authority (is a %s)" % (self.get_hrn(), self.parent.get_hrn(), self.parent.get_type())) + + # Then recurse up the chain - ensure the parent is a trusted + # root or is in the namespace of a trusted root + self.parent.verify_chain(trusted_certs) + else: + # make sure that the trusted root's hrn is a prefix of the child's + trusted_gid = GID(string=trusted_root.save_to_string()) + trusted_type = trusted_gid.get_type() + trusted_hrn = trusted_gid.get_hrn() + #if trusted_type == 'authority': + # trusted_hrn = trusted_hrn[:trusted_hrn.rindex('.')] + cur_hrn = self.get_hrn() + if not hrn_authfor_hrn(trusted_hrn, cur_hrn): + raise GidParentHrn("Trusted root with HRN %s isn't a namespace authority for this cert %s" % (trusted_hrn, cur_hrn)) + + # There are multiple types of authority - accept them all here + if not trusted_type.find('authority') == 0: + raise GidInvalidParentHrn("This cert %s's trusted root signer %s is not an authority (is a %s)" % (self.get_hrn(), trusted_hrn, trusted_type)) + + return