From: Thierry Parmentelat Date: Wed, 8 Feb 2012 12:59:45 +0000 (+0100) Subject: Merge branch 'master' into sqlalchemy X-Git-Tag: sfa-2.1-2~2 X-Git-Url: http://git.onelab.eu/?p=sfa.git;a=commitdiff_plain;h=b7d6a80faf23cb019c74e65c2264e215446f84a3;hp=-c Merge branch 'master' into sqlalchemy Conflicts: sfa/importer/sfa-import-openstack.py sfa/managers/registry_manager.py sfa/openstack/nova_driver.py sfa/storage/PostgreSQL.py sfa/storage/filter.py trashed changes in PostgreSQL / filter applied changes in sfa-import-openstack.py into openstackimporter.py this is mostly manual so please double check --- b7d6a80faf23cb019c74e65c2264e215446f84a3 diff --combined sfa/client/sfi.py index 9ddef9fd,e35a2046..e9614e90 --- a/sfa/client/sfi.py +++ b/sfa/client/sfi.py @@@ -1,8 -1,8 +1,8 @@@ - # + # # sfi.py - basic SFA command-line client # the actual binary in sfa/clientbin essentially runs main() # this module is used in sfascan - # + # import sys sys.path.append('.') @@@ -12,6 -12,7 +12,7 @@@ import socke import datetime import codecs import pickle + import json from lxml import etree from StringIO import StringIO from optparse import OptionParser @@@ -28,8 -29,7 +29,8 @@@ from sfa.util.config import Confi from sfa.util.version import version_core from sfa.util.cache import Cache -from sfa.storage.record import SfaRecord, UserRecord, SliceRecord, NodeRecord, AuthorityRecord +from sfa.storage.model import RegRecord, RegAuthority, RegUser, RegSlice, RegNode +from sfa.storage.model import make_record from sfa.rspecs.rspec import RSpec from sfa.rspecs.rspec_converter import RSpecConverter @@@ -88,16 -88,28 +89,28 @@@ def filter_records(type, records) # save methods - def save_variable_to_file(var, filename, format="text"): - f = open(filename, "w") + def save_raw_to_file(var, filename, format="text", banner=None): + if filename == "-": + # if filename is "-", send it to stdout + f = sys.stdout + else: + f = open(filename, "w") + if banner: + f.write(banner+"\n") if format == "text": f.write(str(var)) elif format == "pickled": f.write(pickle.dumps(var)) + elif format == "json": + if hasattr(json, "dumps"): + f.write(json.dumps(var)) # python 2.6 + else: + f.write(json.write(var)) # python 2.5 else: # this should never happen print "unknown output format", format - + if banner: + f.write('\n'+banner+"\n") def save_rspec_to_file(rspec, filename): if not filename.endswith(".rspec"): @@@ -107,35 -119,44 +120,35 @@@ f.close() return -def save_records_to_file(filename, recordList, format="xml"): +def save_records_to_file(filename, record_dicts, format="xml"): if format == "xml": index = 0 - for record in recordList: + for record_dict in record_dicts: if index > 0: - save_record_to_file(filename + "." + str(index), record) + save_record_to_file(filename + "." + str(index), record_dict) else: - save_record_to_file(filename, record) + save_record_to_file(filename, record_dict) index = index + 1 elif format == "xmllist": f = open(filename, "w") f.write("\n") - for record in recordList: - record = SfaRecord(dict=record) - f.write('\n') + for record_dict in record_dicts: + record_obj=make_record (dict=record_dict) + f.write('\n') f.write("\n") f.close() elif format == "hrnlist": f = open(filename, "w") - for record in recordList: - record = SfaRecord(dict=record) - f.write(record.get_name() + "\n") + for record_dict in record_dicts: + record_obj=make_record (dict=record_dict) + f.write(record_obj.hrn + "\n") f.close() else: # this should never happen print "unknown output format", format -def save_record_to_file(filename, record): - if record['type'] in ['user']: - record = UserRecord(dict=record) - elif record['type'] in ['slice']: - record = SliceRecord(dict=record) - elif record['type'] in ['node']: - record = NodeRecord(dict=record) - elif record['type'] in ['authority', 'ma', 'sa']: - record = AuthorityRecord(dict=record) - else: - record = SfaRecord(dict=record) +def save_record_to_file(filename, record_dict): + rec_record = make_record (dict=record_dict) str = record.save_to_string() f=codecs.open(filename, encoding='utf-8',mode="w") f.write(str) @@@ -146,9 -167,10 +159,9 @@@ # load methods def load_record_from_file(filename): f=codecs.open(filename, encoding="utf-8", mode="r") - str = f.read() + xml_string = f.read() f.close() - record = SfaRecord(string=str) - return record + return make_record (xml=xml_string) import uuid @@@ -289,13 -311,6 +302,6 @@@ class Sfi help="output file format ([xml]|xmllist|hrnlist)", default="xml", choices=("xml", "xmllist", "hrnlist")) - if command in ("status", "version"): - parser.add_option("-o", "--output", dest="file", - help="output dictionary to file", metavar="FILE", default=None) - parser.add_option("-F", "--fileformat", dest="fileformat", type="choice", - help="output file format ([text]|pickled)", default="text", - choices=("text","pickled")) - if command in ("delegate"): parser.add_option("-u", "--user", action="store_true", dest="delegate_user", default=False, @@@ -323,8 -338,13 +329,13 @@@ help="root registry", metavar="URL", default=None) parser.add_option("-s", "--sliceapi", dest="sm", default=None, metavar="URL", help="slice API - in general a SM URL, but can be used to talk to an aggregate") - parser.add_option("-R", "--raw", dest="raw", action="store_true", default=False, - help="Display raw, unparsed server response") + parser.add_option("-R", "--raw", dest="raw", default=None, + help="Save raw, unparsed server response to a file") + parser.add_option("", "--rawformat", dest="rawformat", type="choice", + help="raw file format ([text]|pickled|json)", default="text", + choices=("text","pickled","json")) + parser.add_option("", "--rawbanner", dest="rawbanner", default=None, + help="text string to write before and after raw output") parser.add_option("-d", "--dir", dest="sfi_dir", help="config & working directory - default is %default", metavar="PATH", default=Sfi.default_sfi_dir()) @@@ -644,17 -664,17 +655,17 @@@ else: self.logger.critical("No such registry record file %s"%record) sys.exit(1) - + #========================================================================== # Following functions implement the commands # # Registry-related commands #========================================================================== - + def version(self, options, args): """ - display an SFA server version (GetVersion) + display an SFA server version (GetVersion) or version information about sfi itself """ if options.version_local: @@@ -666,10 -686,11 +677,11 @@@ server = self.sliceapi() result = server.GetVersion() version = ReturnValue.get_value(result) - pprinter = PrettyPrinter(indent=4) - pprinter.pprint(version) - if options.file: - save_variable_to_file(version, options.file, options.fileformat) + if self.options.raw: + save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) + else: + pprinter = PrettyPrinter(indent=4) + pprinter.pprint(version) def list(self, options, args): """ @@@ -701,16 -722,27 +713,16 @@@ self.print_help() sys.exit(1) hrn = args[0] - records = self.registry().Resolve(hrn, self.my_credential_string) - records = filter_records(options.type, records) - if not records: + record_dicts = self.registry().Resolve(hrn, self.my_credential_string) + record_dicts = filter_records(options.type, record_dicts) + if not record_dicts: self.logger.error("No record of type %s"% options.type) + records = [ make_record (dict=record_dict) for record_dict in record_dicts ] for record in records: - if record['type'] in ['user']: - record = UserRecord(dict=record) - elif record['type'] in ['slice']: - record = SliceRecord(dict=record) - elif record['type'] in ['node']: - record = NodeRecord(dict=record) - elif record['type'].startswith('authority'): - record = AuthorityRecord(dict=record) - else: - record = SfaRecord(dict=record) - if (options.format == "text"): - record.dump() - else: - print record.save_to_string() + if (options.format == "text"): record.dump() + else: print record.save_as_xml() if options.file: - save_records_to_file(options.file, records, options.fileformat) + save_records_to_file(options.file, record_dicts, options.fileformat) return def add(self, options, args): @@@ -721,7 -753,7 +733,7 @@@ sys.exit(1) record_filepath = args[0] rec_file = self.get_record_file(record_filepath) - record = load_record_from_file(rec_file).as_dict() + record = load_record_from_file(rec_file).todict() return self.registry().Register(record, auth_cred) def update(self, options, args): @@@ -731,14 -763,14 +743,14 @@@ sys.exit(1) rec_file = self.get_record_file(args[0]) record = load_record_from_file(rec_file) - if record['type'] == "user": - if record.get_name() == self.user: + if record.type == "user": + if record.hrn == self.user: cred = self.my_credential_string else: cred = self.my_authority_credential_string() - elif record['type'] in ["slice"]: + elif record.type in ["slice"]: try: - cred = self.slice_credential_string(record.get_name()) + cred = self.slice_credential_string(record.hrn) except ServerException, e: # XXX smbaker -- once we have better error return codes, update this # to do something better than a string compare @@@ -746,14 -778,14 +758,14 @@@ cred = self.my_authority_credential_string() else: raise - elif record.get_type() in ["authority"]: + elif record.type in ["authority"]: cred = self.my_authority_credential_string() - elif record.get_type() == 'node': + elif record.type == 'node': cred = self.my_authority_credential_string() else: - raise "unknown record type" + record.get_type() - record = record.as_dict() - return self.registry().Update(record, cred) + raise "unknown record type" + record.type + record_dict = record.todict() + return self.registry().Update(record_dict, cred) def remove(self, options, args): "remove registry record by name (Remove)" @@@ -785,11 -817,11 +797,11 @@@ result = server.ListSlices(creds, *self.ois(server,api_options)) value = ReturnValue.get_value(result) if self.options.raw: - print result + save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) else: display_list(value) return - + # show rspec for named slice def resources(self, options, args): """ @@@ -806,9 -838,9 +818,9 @@@ or with an slice hrn, shows currently p creds.append(self.my_credential_string) if options.delegate: creds.append(self.delegate_cred(cred, get_authority(self.authority))) - + # no need to check if server accepts the options argument since the options has - # been a required argument since v1 API + # been a required argument since v1 API api_options = {} # always send call_id to v2 servers api_options ['call_id'] = unique_call_id() @@@ -831,18 -863,18 +843,18 @@@ # just request the version the client wants api_options['geni_rspec_version'] = version_manager.get_version(options.rspec_version).to_dict() else: - api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3.0'} + api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3.0'} else: - api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3.0'} + api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3.0'} result = server.ListResources (creds, api_options) value = ReturnValue.get_value(result) - if options.file is None: - if self.options.raw: - print result - else: - display_rspec(value, options.format) - else: + if self.options.raw: + save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) + if options.file is not None: save_rspec_to_file(value, options.file) + if (self.options.raw is None) and (options.file is None): + display_rspec(value, options.format) + return def create(self, options, args): @@@ -868,8 -900,8 +880,8 @@@ # delegated_cred = self.delegate_cred(slice_cred, server_version['hrn']) #elif server_version.get('urn'): # delegated_cred = self.delegate_cred(slice_cred, urn_to_hrn(server_version['urn'])) - - # rspec + + # rspec rspec_file = self.get_rspec_file(args[1]) rspec = open(rspec_file).read() @@@ -894,8 -926,8 +906,8 @@@ rspec = RSpecConverter.to_pg_rspec(rspec.toxml(), content_type='request') else: users = sfa_users_arg(user_records, slice_record) - - # do not append users, keys, or slice tags. Anything + + # do not append users, keys, or slice tags. Anything # not contained in this request will be removed from the slice # CreateSliver has supported the options argument for a while now so it should @@@ -906,13 -938,13 +918,13 @@@ result = server.CreateSliver(slice_urn, creds, rspec, users, *self.ois(server, api_options)) value = ReturnValue.get_value(result) - if options.file is None: - if self.options.raw: - print result - else: - print value - else: + if self.options.raw: + save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) + if options.file is not None: save_rspec_to_file (value, options.file) + if (self.options.raw is None) and (options.file is None): + print value + return value def delete(self, options, args): @@@ -938,7 -970,7 +950,7 @@@ result = server.DeleteSliver(slice_urn, creds, *self.ois(server, api_options ) ) value = ReturnValue.get_value(result) if self.options.raw: - print result + save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) else: print value return value @@@ -966,11 -998,9 +978,9 @@@ result = server.SliverStatus(slice_urn, creds, *self.ois(server,api_options)) value = ReturnValue.get_value(result) if self.options.raw: - print result + save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) else: print value - if options.file: - save_variable_to_file(value, options.file, options.fileformat) def start(self, options, args): """ @@@ -992,7 -1022,7 +1002,7 @@@ result = server.Start(slice_urn, creds) value = ReturnValue.get_value(result) if self.options.raw: - print result + save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) else: print value return value @@@ -1014,7 -1044,7 +1024,7 @@@ result = server.Stop(slice_urn, creds) value = ReturnValue.get_value(result) if self.options.raw: - print result + save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) else: print value return value @@@ -1037,7 -1067,7 +1047,7 @@@ result = server.reset_slice(creds, slice_urn) value = ReturnValue.get_value(result) if self.options.raw: - print result + save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) else: print value return value @@@ -1064,7 -1094,7 +1074,7 @@@ result = server.RenewSliver(slice_urn, creds, time, *self.ois(server,api_options)) value = ReturnValue.get_value(result) if self.options.raw: - print result + save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) else: print value return value @@@ -1087,7 -1117,7 +1097,7 @@@ result = server.Shutdown(slice_urn, creds) value = ReturnValue.get_value(result) if self.options.raw: - print result + save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) else: print value return value diff --combined sfa/generic/openstack.py index 024f2914,590bb573..bac57dc7 --- a/sfa/generic/openstack.py +++ b/sfa/generic/openstack.py @@@ -1,30 -1,35 +1,30 @@@ from sfa.generic import Generic import sfa.server.sfaapi - import sfa.openstack.openstack_driver + import sfa.openstack.nova_driver import sfa.managers.registry_manager_openstack + import sfa.managers.aggregate_manager import sfa.managers.slice_manager - import sfa.managers.aggregate_manager_openstack -class openstack (Generic): - - # use the standard api class - def api_class (self): - return sfa.server.sfaapi.SfaApi +# use pl as a model so we only redefine what's different +from sfa.generic.pl import pl +class openstack (pl): + + # the importer class + def importer_class (self): + import sfa.importer.openstackimporter + return sfa.importer.openstackimporter.OpenstackImporter + # the manager classes for the server-side services def registry_manager_class (self) : return sfa.managers.registry_manager_openstack.RegistryManager - def slicemgr_manager_class (self) : - return sfa.managers.slice_manager.SliceManager def aggregate_manager_class (self) : - return sfa.managers.aggregate_manager_openstack.AggregateManager + return sfa.managers.aggregate_manager.AggregateManager # driver class for server-side services, talk to the whole testbed def driver_class (self): - return sfa.openstack.openstack_driver.OpenstackDriver + return sfa.openstack.nova_driver.NovaDriver - # for the component mode, to be run on board planetlab nodes - # manager class - def component_manager_class (self): - return sfa.managers.component_manager_pl - # driver_class - def component_driver_class (self): - return sfa.plc.plcomponentdriver.PlComponentDriver diff --combined sfa/importer/openstackimporter.py index 844980de,00000000..2bf1da31 mode 100644,000000..100644 --- a/sfa/importer/openstackimporter.py +++ b/sfa/importer/openstackimporter.py @@@ -1,143 -1,0 +1,143 @@@ +import os + +from sfa.util.config import Config +from sfa.util.xrn import Xrn, get_leaf, get_authority, hrn_to_urn +from sfa.util.plxrn import hostname_to_hrn, slicename_to_hrn, email_to_hrn, hrn_to_pl_slicename + +from sfa.trust.gid import create_uuid +from sfa.trust.certificate import convert_public_key, Keypair + +from sfa.storage.alchemy import dbsession +from sfa.storage.model import RegRecord, RegAuthority, RegUser, RegSlice, RegNode + - from sfa.openstack.openstack_shell import OpenstackShell ++from sfa.openstack.nova_shell import NovaShell + +def load_keys(filename): + keys = {} + tmp_dict = {} + try: + execfile(filename, tmp_dict) + if 'keys' in tmp_dict: + keys = tmp_dict['keys'] + return keys + except: + return keys + +def save_keys(filename, keys): + f = open(filename, 'w') + f.write("keys = %s" % str(keys)) + f.close() + +class OpenstackImporter: + + def __init__ (self, auth_hierarchy, logger): + self.auth_hierarchy = auth_hierarchy + self.logger=logger + + def add_options (self, parser): + self.logger.debug ("OpenstackImporter: no options yet") + pass + + def run (self, options): + # we don't have any options for now + self.logger.info ("PlImporter.run : to do") + + config = Config () + interface_hrn = config.SFA_INTERFACE_HRN + root_auth = config.SFA_REGISTRY_ROOT_AUTH - shell = OpenstackShell (config) ++ shell = NovaShell (config) + + # create dict of all existing sfa records + existing_records = {} + existing_hrns = [] + key_ids = [] + for record in dbsession.query(RegRecord): + existing_records[ (record.hrn, record.type,) ] = record + existing_hrns.append(record.hrn) + + # Get all users + persons = shell.user_get_all() + persons_dict = {} + keys_filename = config.config_path + os.sep + 'person_keys.py' + old_person_keys = load_keys(keys_filename) + person_keys = {} + for person in persons: + hrn = config.SFA_INTERFACE_HRN + "." + person.id + persons_dict[hrn] = person + old_keys = old_person_keys.get(person.id, []) + keys = [k.public_key for k in shell.key_pair_get_all_by_user(person.id)] + person_keys[person.id] = keys + update_record = False + if old_keys != keys: + update_record = True + if hrn not in existing_hrns or \ + (hrn, 'user') not in existing_records or update_record: + urn = hrn_to_urn(hrn, 'user') + + if keys: + try: + pkey = convert_public_key(keys[0]) + except: + logger.log_exc('unable to convert public key for %s' % hrn) + pkey = Keypair(create=True) + else: + logger.warn("OpenstackImporter: person %s does not have a PL public key"%hrn) + pkey = Keypair(create=True) + person_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey) + person_record = RegUser () + person_record.type='user' + person_record.hrn=hrn + person_record.gid=person_gid + person_record.authority=get_authority(hrn) + dbsession.add(person_record) + dbsession.commit() + logger.info("OpenstackImporter: imported person %s" % person_record) + + # Get all projects + projects = shell.project_get_all() + projects_dict = {} + for project in projects: + hrn = config.SFA_INTERFACE_HRN + '.' + project.id + projects_dict[hrn] = project + if hrn not in existing_hrns or \ + (hrn, 'slice') not in existing_records: + pkey = Keypair(create=True) + urn = hrn_to_urn(hrn, 'slice') + project_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey) + project_record = RegSlice () + project_record.type='slice' + project_record.hrn=hrn + project_record.gid=project_gid + project_record.authority=get_authority(hrn) + dbsession.add(project_record) + dbsession.commit() + logger.info("OpenstackImporter: imported slice: %s" % project_record) + + # remove stale records + system_records = [interface_hrn, root_auth, interface_hrn + '.slicemanager'] + for (record_hrn, type) in existing_records.keys(): + if record_hrn in system_records: + continue + + record = existing_records[(record_hrn, type)] + if record.peer_authority: + continue + + if type == 'user': + if record_hrn in persons_dict: + continue + elif type == 'slice': + if record_hrn in projects_dict: + continue + else: + continue + + record_object = existing_records[ (record_hrn, type) ] + logger.info("OpenstackImporter: removing %s " % record) + dbsession.delete(record_object) + dbsession.commit() + + # save pub keys + logger.info('OpenstackImporter: saving current pub keys') + save_keys(keys_filename, person_keys) + diff --combined sfa/managers/registry_manager.py index ddca3ae5,9fd26597..f07b5615 --- a/sfa/managers/registry_manager.py +++ b/sfa/managers/registry_manager.py @@@ -1,4 -1,5 +1,4 @@@ import types -import time # for get_key_from_incoming_ip import tempfile import os @@@ -18,8 -19,8 +18,8 @@@ from sfa.trust.credential import Creden from sfa.trust.certificate import Certificate, Keypair, convert_public_key from sfa.trust.gid import create_uuid -from sfa.storage.record import SfaRecord -from sfa.storage.table import SfaTable +from sfa.storage.model import make_record, RegRecord, RegUser +from sfa.storage.alchemy import dbsession class RegistryManager: @@@ -48,19 -49,19 +48,19 @@@ auth_hrn = api.auth.get_authority(hrn) if not auth_hrn or hrn == api.config.SFA_INTERFACE_HRN: auth_hrn = hrn - # get record info auth_info = api.auth.get_auth_info(auth_hrn) - table = SfaTable() - records = table.findObjects({'type': type, 'hrn': hrn}) - if not records: - raise RecordNotFound(hrn) - record = records[0] + # get record info + record=dbsession.query(RegRecord).filter_by(type=type,hrn=hrn).first() + if not record: + raise RecordNotFound("hrn=%s, type=%s"%(hrn,type)) # verify_cancreate_credential requires that the member lists # (researchers, pis, etc) be filled in - self.driver.augment_records_with_testbed_info (record) - if not self.driver.is_enabled (record): - raise AccountNotEnabled(": PlanetLab account %s is not enabled. Please contact your site PI" %(record['email'])) + logger.debug("get credential before augment dict, keys=%s"%record.__dict__.keys()) + self.driver.augment_records_with_testbed_info (record.__dict__) + logger.debug("get credential after augment dict, keys=%s"%record.__dict__.keys()) + if not self.driver.is_enabled (record.__dict__): + raise AccountNotEnabled(": PlanetLab account %s is not enabled. Please contact your site PI" %(record.email)) # get the callers gid # if this is a self cred the record's gid is the caller's gid @@@ -72,12 -73,12 +72,12 @@@ caller_hrn = caller_gid.get_hrn() object_hrn = record.get_gid_object().get_hrn() - rights = api.auth.determine_user_rights(caller_hrn, record) + rights = api.auth.determine_user_rights(caller_hrn, record.__dict__) # make sure caller has rights to this object if rights.is_empty(): - raise PermissionError(caller_hrn + " has no rights to " + record.hrn) - + raise PermissionError("%s has no rights to %s (%s)" % \ + (caller_hrn, object_hrn, xrn)) - object_gid = GID(string=record['gid']) + object_gid = GID(string=record.gid) new_cred = Credential(subject = object_gid.get_subject()) new_cred.set_gid_caller(caller_gid) new_cred.set_gid_object(object_gid) @@@ -85,8 -86,8 +85,8 @@@ #new_cred.set_pubkey(object_gid.get_pubkey()) new_cred.set_privileges(rights) new_cred.get_privileges().delegate_all_privileges(True) - if 'expires' in record: - date = utcparse(record['expires']) + if hasattr(record,'expires'): + date = utcparse(record.expires) expires = datetime_to_epoch(date) new_cred.set_expiration(int(expires)) auth_kind = "authority,ma,sa" @@@ -106,11 -107,10 +106,11 @@@ if not type: type = Xrn(xrns).get_type() hrns = [urn_to_hrn(xrn)[0] for xrn in xrns] + # load all known registry names into a prefix tree and attempt to find # the longest matching prefix - # create a dict where key is a registry hrn and its value is a - # hrns at that registry (determined by the known prefix tree). + # create a dict where key is a registry hrn and its value is a list + # of hrns at that registry (determined by the known prefix tree). xrn_dict = {} registries = api.registries tree = prefixTree() @@@ -136,52 -136,47 +136,52 @@@ credential = api.getCredential() interface = api.registries[registry_hrn] server_proxy = api.server_proxy(interface, credential) - peer_records = server_proxy.Resolve(xrns, credential) - records.extend([SfaRecord(dict=record).as_dict() for record in peer_records]) + peer_records = server_proxy.Resolve(xrns, credential,type) + # pass foreign records as-is + # previous code used to read + # records.extend([SfaRecord(dict=record).as_dict() for record in peer_records]) + # not sure why the records coming through xmlrpc had to be processed at all + records.extend(peer_records) # try resolving the remaining unfound records at the local registry local_hrns = list ( set(hrns).difference([record['hrn'] for record in records]) ) # - table = SfaTable() - local_records = table.findObjects({'hrn': local_hrns}) + local_records = dbsession.query(RegRecord).filter(RegRecord.hrn.in_(local_hrns)) + if type: + local_records = local_records.filter_by(type=type) + local_records=local_records.all() + logger.info("Resolve: local_records=%s (type=%s)"%(local_records,type)) + local_dicts = [ record.__dict__ for record in local_records ] if full: # in full mode we get as much info as we can, which involves contacting the # testbed for getting implementation details about the record - self.driver.augment_records_with_testbed_info(local_records) + self.driver.augment_records_with_testbed_info(local_dicts) # also we fill the 'url' field for known authorities # used to be in the driver code, sounds like a poorman thing though def solve_neighbour_url (record): - if not record['type'].startswith('authority'): return - hrn=record['hrn'] + if not record.type.startswith('authority'): return + hrn=record.hrn for neighbour_dict in [ api.aggregates, api.registries ]: if hrn in neighbour_dict: - record['url']=neighbour_dict[hrn].get_url() + record.url=neighbour_dict[hrn].get_url() return - [ solve_neighbour_url (record) for record in local_records ] - + for record in local_records: solve_neighbour_url (record) - - # convert local record objects to dicts - records.extend([dict(record) for record in local_records]) - if type: - records = filter(lambda rec: rec['type'] in [type], records) - + # convert local record objects to dicts for xmlrpc + # xxx somehow here calling dict(record) issues a weird error + # however record.todict() seems to work fine + # records.extend( [ dict(record) for record in local_records ] ) + records.extend( [ record.todict() for record in local_records ] ) if not records: raise RecordNotFound(str(hrns)) return records - def List(self, api, xrn, origin_hrn=None): + def List (self, api, xrn, origin_hrn=None): hrn, type = urn_to_hrn(xrn) # load all know registry names into a prefix tree and attempt to find # the longest matching prefix - records = [] registries = api.registries registry_hrns = registries.keys() tree = prefixTree() @@@ -193,24 -188,23 +193,24 @@@ raise MissingAuthority(xrn) # if the best match (longest matching hrn) is not the local registry, # forward the request - records = [] + record_dicts = [] if registry_hrn != api.hrn: credential = api.getCredential() interface = api.registries[registry_hrn] server_proxy = api.server_proxy(interface, credential) record_list = server_proxy.List(xrn, credential) - records = [SfaRecord(dict=record).as_dict() for record in record_list] + # same as above, no need to process what comes from through xmlrpc + # pass foreign records as-is + record_dicts = record_list # if we still have not found the record yet, try the local registry - if not records: + if not record_dicts: if not api.auth.hierarchy.auth_exists(hrn): raise MissingAuthority(hrn) + records = dbsession.query(RegRecord).filter_by(authority=hrn) + record_dicts=[ record.todict() for record in records ] - table = SfaTable() - records = table.find({'authority': hrn}) - - return records + return record_dicts def CreateGid(self, api, xrn, cert): @@@ -233,58 -227,57 +233,58 @@@ # subject_record describes the subject of the relationships # ref_record contains the target values for the various relationships we need to manage # (to begin with, this is just the slice x person relationship) - def update_relations (self, subject_record, ref_record): - type=subject_record['type'] + def update_relations (self, subject_obj, ref_obj): + type=subject_obj.type if type=='slice': - self.update_relation(subject_record, 'researcher', ref_record.get('researcher'), 'user') + self.update_relation(subject_obj, 'researcher', ref_obj.researcher, 'user') # field_key is the name of one field in the record, typically 'researcher' for a 'slice' record # hrns is the list of hrns that should be linked to the subject from now on # target_type would be e.g. 'user' in the 'slice' x 'researcher' example - def update_relation (self, sfa_record, field_key, hrns, target_type): + def update_relation (self, record_obj, field_key, hrns, target_type): # locate the linked objects in our db - subject_type=sfa_record['type'] - subject_id=sfa_record['pointer'] - table = SfaTable() - link_sfa_records = table.find ({'type':target_type, 'hrn': hrns}) - link_ids = [ rec.get('pointer') for rec in link_sfa_records ] + subject_type=record_obj.type + subject_id=record_obj.pointer + # get the 'pointer' field of all matching records + link_id_tuples = dbsession.query(RegRecord.pointer).filter_by(type=target_type).filter(RegRecord.hrn.in_(hrns)).all() + # sqlalchemy returns named tuples for columns + link_ids = [ tuple.pointer for tuple in link_id_tuples ] self.driver.update_relation (subject_type, target_type, subject_id, link_ids) - - def Register(self, api, record): + def Register(self, api, record_dict): - hrn, type = record['hrn'], record['type'] + hrn, type = record_dict['hrn'], record_dict['type'] urn = hrn_to_urn(hrn,type) # validate the type if type not in ['authority', 'slice', 'node', 'user']: raise UnknownSfaType(type) - # check if record already exists - table = SfaTable() - existing_records = table.find({'type': type, 'hrn': hrn}) + # check if record_dict already exists + existing_records = dbsession.query(RegRecord).filter_by(type=type,hrn=hrn).all() if existing_records: raise ExistingRecord(hrn) - record = SfaRecord(dict = record) - record['authority'] = get_authority(record['hrn']) - auth_info = api.auth.get_auth_info(record['authority']) + assert ('type' in record_dict) + # returns the right type of RegRecord according to type in record + record = make_record(dict=record_dict) + record.just_created() + record.authority = get_authority(record.hrn) + auth_info = api.auth.get_auth_info(record.authority) pub_key = None # make sure record has a gid - if 'gid' not in record: + if not record.gid: uuid = create_uuid() pkey = Keypair(create=True) - if 'keys' in record and record['keys']: - pub_key=record['keys'] + if getattr(record,'keys',None): + pub_key=record.keys # use only first key in record - if isinstance(record['keys'], types.ListType): - pub_key = record['keys'][0] + if isinstance(record.keys, types.ListType): + pub_key = record.keys[0] pkey = convert_public_key(pub_key) gid_object = api.auth.hierarchy.create_gid(urn, uuid, pkey) gid = gid_object.save_to_string(save_parents=True) - record['gid'] = gid - record.set_gid(gid) + record.gid = gid if type in ["authority"]: # update the tree @@@ -293,18 -286,14 +293,18 @@@ # get the GID from the newly created authority gid = auth_info.get_gid_object() - record.set_gid(gid.save_to_string(save_parents=True)) + record.gid=gid.save_to_string(save_parents=True) + # post-process / cleanup for relation ships + if isinstance (record, RegUser): + record.normalize_xml() + # update testbed-specific data if needed - pointer = self.driver.register (record, hrn, pub_key) + pointer = self.driver.register (record.__dict__, hrn, pub_key) - record.set_pointer(pointer) - record_id = table.insert(record) - record['record_id'] = record_id + record.pointer=pointer + dbsession.add(record) + dbsession.commit() # update membership for researchers, pis, owners, operators self.update_relations (record, record) @@@ -312,16 -301,17 +312,16 @@@ return record.get_gid_object().save_to_string(save_parents=True) def Update(self, api, record_dict): - new_record = SfaRecord(dict = record_dict) - type = new_record['type'] - hrn = new_record['hrn'] - urn = hrn_to_urn(hrn,type) - table = SfaTable() + assert ('type' in record_dict) + new_record=RegRecord(dict=record_dict) + type = new_record.type + hrn = new_record.hrn + # make sure the record exists - records = table.findObjects({'type': type, 'hrn': hrn}) - if not records: - raise RecordNotFound(hrn) - record = records[0] - record['last_updated'] = time.gmtime() + record = dbsession.query(RegRecord).filter_by(type=type,hrn=hrn).first() + if not record: + raise RecordNotFound("hrn=%s, type=%s"%(hrn,type)) + record.just_updated() # validate the type if type not in ['authority', 'slice', 'node', 'user']: @@@ -329,18 -319,18 +329,18 @@@ # Use the pointer from the existing record, not the one that the user # gave us. This prevents the user from inserting a forged pointer - pointer = record['pointer'] + pointer = record.pointer # is the a change in keys ? new_key=None if type=='user': - if 'keys' in new_record and new_record['keys']: - new_key=new_record['keys'] + if getattr(new_key,'keys',None): + new_key=new_record.keys if isinstance (new_key,types.ListType): new_key=new_key[0] # update the PLC information that was specified with the record - if not self.driver.update (record, new_record, hrn, new_key): + if not self.driver.update (record.__dict__, new_record.__dict__, hrn, new_key): logger.warning("driver.update failed") # take new_key into account @@@ -348,11 -338,11 +348,11 @@@ # update the openssl key and gid pkey = convert_public_key(new_key) uuid = create_uuid() + urn = hrn_to_urn(hrn,type) gid_object = api.auth.hierarchy.create_gid(urn, uuid, pkey) gid = gid_object.save_to_string(save_parents=True) - record['gid'] = gid - record = SfaRecord(dict=record) - table.update(record) + record.gid = gid + dsession.commit() # update membership for researchers, pis, owners, operators self.update_relations (record, new_record) @@@ -361,19 -351,19 +361,19 @@@ # expecting an Xrn instance def Remove(self, api, xrn, origin_hrn=None): - - table = SfaTable() - filter = {'hrn': xrn.get_hrn()} hrn=xrn.get_hrn() type=xrn.get_type() + request=dbsession.query(RegRecord).filter_by(hrn=hrn) if type and type not in ['all', '*']: - filter['type'] = type + request=request.filter_by(type=type) - records = table.find(filter) - if not records: raise RecordNotFound(hrn) - record = records[0] - type = record['type'] - + record = request.first() + if not record: + msg="Could not find hrn %s"%hrn + if type: msg += " type=%s"%type + raise RecordNotFound(msg) + + type = record.type if type not in ['slice', 'user', 'node', 'authority'] : raise UnknownSfaType(type) @@@ -392,16 -382,15 +392,16 @@@ # call testbed callback first # IIUC this is done on the local testbed TOO because of the refreshpeer link - if not self.driver.remove(record): + if not self.driver.remove(record.__dict__): logger.warning("driver.remove failed") # delete from sfa db - table.remove(record) + dbsession.delete(record) + dbsession.commit() return 1 - # This is a PLC-specific thing... + # This is a PLC-specific thing, won't work with other platforms def get_key_from_incoming_ip (self, api): # verify that the callers's ip address exist in the db and is an interface # for a node in the db @@@ -415,20 -404,23 +415,20 @@@ node = nodes[0] # look up the sfa record - table = SfaTable() - records = table.findObjects({'type': 'node', 'pointer': node['node_id']}) - if not records: - raise RecordNotFound("pointer:" + str(node['node_id'])) - record = records[0] + record=dbsession.query(RegRecord).filter_by(type='node',pointer=node['node_id']).first() + if not record: + raise RecordNotFound("node with pointer %s"%node['node_id']) # generate a new keypair and gid uuid = create_uuid() pkey = Keypair(create=True) - urn = hrn_to_urn(record['hrn'], record['type']) + urn = hrn_to_urn(record.hrn, record.type) gid_object = api.auth.hierarchy.create_gid(urn, uuid, pkey) gid = gid_object.save_to_string(save_parents=True) - record['gid'] = gid - record.set_gid(gid) + record.gid = gid # update the record - table.update(record) + dbsession.commit() # attempt the scp the key # and gid onto the node diff --combined sfa/openstack/nova_driver.py index 368b4080,0ee4c727..404021be --- a/sfa/openstack/nova_driver.py +++ b/sfa/openstack/nova_driver.py @@@ -3,7 -3,6 +3,7 @@@ import datetim # from sfa.util.faults import MissingSfaInfo, UnknownSfaType, \ RecordNotFound, SfaNotImplemented, SliverDoesNotExist + from sfa.util.sfalogging import logger from sfa.util.defaultdict import defaultdict from sfa.util.sfatime import utcparse, datetime_to_string, datetime_to_epoch @@@ -11,14 -10,11 +11,13 @@@ from sfa.util.xrn import Xrn, hrn_to_ur from sfa.util.cache import Cache # used to be used in get_ticket #from sfa.trust.sfaticket import SfaTicket + from sfa.rspecs.version_manager import VersionManager from sfa.rspecs.rspec import RSpec + # the driver interface, mostly provides default behaviours from sfa.managers.driver import Driver - - from sfa.openstack.openstack_shell import OpenstackShell + from sfa.openstack.nova_shell import NovaShell from sfa.openstack.osaggregate import OSAggregate from sfa.plc.plslices import PlSlices from sfa.util.osxrn import OSXrn @@@ -36,19 -32,19 +35,19 @@@ def list_to_dict(recs, key) # can be sent as-is; it takes care of authentication # from the global config # - class OpenstackDriver (Driver): + class NovaDriver (Driver): # the cache instance is a class member so it survives across incoming requests cache = None def __init__ (self, config): Driver.__init__ (self, config) - self.shell = OpenstackShell (config) + self.shell = NovaShell (config) self.cache=None if config.SFA_AGGREGATE_CACHING: - if OpenstackDriver.cache is None: - OpenstackDriver.cache = Cache() - self.cache = OpenstackDriver.cache + if NovaDriver.cache is None: + NovaDriver.cache = Cache() + self.cache = NovaDriver.cache ######################################## ########## registry oriented @@@ -122,28 -118,29 +121,29 @@@ name = Xrn(record['hrn']).get_leaf() os_record = None if record['type'] == 'user': - os_record = self.shell.user_get(name) + os_record = self.shell.auth_manager.get_user(name) + projects = self.shell.db.project_get_by_user(name) record['slices'] = [self.hrn + "." + proj.name for \ - proj in os_record.projects] - record['roles'] = [role for role in os_record.roles] - keys = self.shell.key_pair_get_all_by_user(name) + proj in projects] + record['roles'] = self.shell.db.user_get_roles(name) + keys = self.shell.db.key_pair_get_all_by_user(name) record['keys'] = [key.public_key for key in keys] elif record['type'] == 'slice': - os_record = self.shell.project_get(name) + os_record = self.shell.auth_manager.get_project(name) record['description'] = os_record.description - record['PI'] = self.hrn + "." + os_record.project_manager + record['PI'] = [self.hrn + "." + os_record.project_manager.name] record['geni_creator'] = record['PI'] - record['researcher'] = [self.hrn + "." + user.name for \ - user in os_record.members] + record['researcher'] = [self.hrn + "." + user for \ + user in os_record.member_ids] else: continue record['geni_urn'] = hrn_to_urn(record['hrn'], record['type']) record['geni_certificate'] = record['gid'] record['name'] = os_record.name - if os_record.created_at is not None: - record['date_created'] = datetime_to_string(utcparse(os_record.created_at)) - if os_record.updated_at is not None: - record['last_updated'] = datetime_to_string(utcparse(os_record.updated_at)) + #if os_record.created_at is not None: + # record['date_created'] = datetime_to_string(utcparse(os_record.created_at)) + #if os_record.updated_at is not None: + # record['last_updated'] = datetime_to_string(utcparse(os_record.updated_at)) return records @@@ -199,8 -196,8 +199,8 @@@ return slices # get data from db - slices = self.shell.project_get_all() - slice_urns = [OSXrn(name, 'slice').urn for name in slice] + projs = self.shell.auth_manager.get_projects() + slice_urns = [OSXrn(proj.name, 'slice').urn for proj in projs] # cache the result if self.cache: @@@ -295,38 -292,20 +295,20 @@@ def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, users, options): - aggregate = PlAggregate(self) - slices = PlSlices(self) - peer = slices.get_peer(slice_hrn) - sfa_peer = slices.get_sfa_peer(slice_hrn) - slice_record=None - if users: - slice_record = users[0].get('slice_record', {}) - + aggregate = OSAggregate(self) + slicename = get_leaf(slice_hrn) + # parse rspec rspec = RSpec(rspec_string) requested_attributes = rspec.version.get_slice_attributes() - # ensure site record exists - site = slices.verify_site(slice_hrn, slice_record, peer, sfa_peer, options=options) # ensure slice record exists - slice = slices.verify_slice(slice_hrn, slice_record, peer, sfa_peer, options=options) + slice = aggregate.verify_slice(slicename, users, options=options) # ensure person records exists - persons = slices.verify_persons(slice_hrn, slice, users, peer, sfa_peer, options=options) - # ensure slice attributes exists - slices.verify_slice_attributes(slice, requested_attributes, options=options) - + persons = aggregate.verify_slice_users(slicename, users, options=options) # add/remove slice from nodes - requested_slivers = [node.get('component_name') for node in rspec.version.get_nodes_with_slivers()] - nodes = slices.verify_slice_nodes(slice, requested_slivers, peer) + slices.verify_instances(slicename, rspec) - # add/remove links links - slices.verify_slice_links(slice, rspec.version.get_link_requests(), nodes) - - # handle MyPLC peer association. - # only used by plc and ple. - slices.handle_peer(site, slice, persons, peer) - return aggregate.get_rspec(slice_xrn=slice_urn, version=rspec.version) def delete_sliver (self, slice_urn, slice_hrn, creds, options): @@@ -336,9 -315,9 +318,9 @@@ return 1 self.shell.DeleteSliceFromNodes(slicename, slice['node_ids']) - instances = self.shell.instance_get_all_by_project(name) + instances = self.shell.db.instance_get_all_by_project(name) for instance in instances: - self.shell.instance_destroy(instance.instance_id) + self.shell.db.instance_destroy(instance.instance_id) return 1 def renew_sliver (self, slice_urn, slice_hrn, creds, expiration_time, options): @@@ -349,14 -328,10 +331,10 @@@ def stop_slice (self, slice_urn, slice_hrn, creds): name = OSXrn(xrn=slice_urn).name - slice = self.shell.project_get(name) - if not slice: - return 1 - - self.shell.DeleteSliceFromNodes(slicename, slice['node_ids']) - instances = self.shell.instance_get_all_by_project(name) + slice = self.shell.get_project(name) + instances = self.shell.db.instance_get_all_by_project(name) for instance in instances: - self.shell.instance_stop(instance.instance_id) + self.shell.db.instance_stop(instance.instance_id) return 1 def reset_slice (self, slice_urn, slice_hrn, creds): diff --combined sfa/util/xrn.py index fb9e864f,90d8549f..f48a3773 --- a/sfa/util/xrn.py +++ b/sfa/util/xrn.py @@@ -33,7 -33,7 +33,7 @@@ def hrn_to_urn(hrn,type): return Xrn(hr def hrn_authfor_hrn(parenthrn, hrn): return Xrn.hrn_is_auth_for_hrn(parenthrn, hrn) def urn_to_sliver_id(urn, slice_id, node_id, index=0): - return ":".join(map(str, [urn, slice_id, node_id, index])) + return Xrn(urn).get_sliver_id(slice_id, node_id, index) class Xrn: @@@ -130,13 -130,6 +130,13 @@@ # if not type: # debug_logger.debug("type-less Xrn's are not safe") + def __repr__ (self): + result="