From: Sandrine Avakian Date: Tue, 13 Sep 2011 15:43:50 +0000 (+0200) Subject: Fixed symboli links problem X-Git-Tag: sfa-2.1-24~3^2~336 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=68649f7cebdff2ea36a08cd904e57de34b7b1682;p=sfa.git Fixed symboli links problem --- diff --git a/sfa/senslab/OARrestapi.py b/sfa/senslab/OARrestapi.py deleted file mode 120000 index 784adc8d..00000000 --- a/sfa/senslab/OARrestapi.py +++ /dev/null @@ -1 +0,0 @@ -/home/avakian/Documents/sfa/sfa/senslab/OARrestapi.py \ No newline at end of file diff --git a/sfa/senslab/OARrspec.py b/sfa/senslab/OARrspec.py deleted file mode 120000 index f8ead80c..00000000 --- a/sfa/senslab/OARrspec.py +++ /dev/null @@ -1 +0,0 @@ -/home/avakian/Documents/sfa/sfa/senslab/OARrspec.py \ No newline at end of file diff --git a/sfa/senslab/OARrspec.py b/sfa/senslab/OARrspec.py new file mode 100644 index 00000000..a045e475 --- /dev/null +++ b/sfa/senslab/OARrspec.py @@ -0,0 +1,123 @@ +########################################################################### +# Copyright (C) 2011 by root +# +# +# Copyright: See COPYING file that comes with this distribution +# +########################################################################### +#!/usr/bin/python + +# import modules used here -- sys is a very standard one +import sys +import httplib +import json + + +from sfa.util.xrn import * +from sfa.util.plxrn import * +from sfa.rspecs.sfa_rspec import SfaRSpec +from sfa.rspecs.pg_rspec import PGRSpec +from sfa.rspecs.rspec_version import RSpecVersion + +from sfa.senslab.OARrestapi import * + +class OARrspec: + + + sites = {} + nodes = {} + + prepared=False + #panos new user options variable + user_options = {} + + def __init__(self): + self.OARImporter = OARapi() + print >>sys.stderr,'\r\n \r\n \t\t__INIT OARRSPEC__' + + + def prepare_sites(self, force=False): + print >>sys.stderr,'\r\n \r\n ++++++++++++++\t\t', self.OARImporter.GetSites() + if not self.sites or force: + for site in self.OARImporter.GetSites(): + print >>sys.stderr,'prepare_sites : site ', site + self.sites[site['site_id']] = site + + + def prepare_nodes(self, force=False): + if not self.nodes or force: + for node in self.OARImporter.GetNodes(): + self.nodes[node['node_id']] = node + print >>sys.stderr,'prepare_nodes:node', node + + + + #def prepare_node_tags(self, force=False): + #if not self.node_tags or force: + #for node_tag in self.api.plshell.GetNodeTags(self.api.plauth): + #self.node_tags[node_tag['node_tag_id']] = node_tag + + + def prepare(self, force=False): + if not self.prepared or force: + self.prepare_sites(force) + self.prepare_nodes(force) + + # add site/interface info to nodes + for node_id in self.nodes: + node = self.nodes[node_id] + site = self.sites[node['site_id']] + + node['network'] = "grenoble-senslab" + node['network_urn'] = hrn_to_urn(node['network'], 'authority+sa') + node['urn'] = hostname_to_urn(node['network'], site['login_base'], node['hostname']) + node['site_urn'] = hrn_to_urn(PlXrn.site_hrn(node['network'], site['login_base']), 'authority') + node['site'] = site + + #node['tags'] = tags + print >>sys.stderr, "\r\n OAR prepare ", node + self.prepared = True + + def get_rspec(self, slice_xrn=None, version = None): + print>>sys.stderr, " \r\n OARrspec \t\t get_spec **************\r\n" + self.prepare() + + rspec = None + rspec_version = RSpecVersion(version) + print >>sys.stderr, '\r\n \t\t rspec_version type', rspec_version['type'] + if rspec_version['type'].lower() == 'sfa': + rspec = SfaRSpec("",{},self.user_options) + else: + rspec = SfaRSpec("",{},self.user_options) + + + rspec.add_nodes(self.nodes.values()) + print >>sys.stderr, 'after add_nodes' + + + #rspec.add_links(self.links.values()) + + #if slice_xrn: + ## get slice details + #slice_hrn, _ = urn_to_hrn(slice_xrn) + #slice_name = hrn_to_pl_slicename(slice_hrn) + #slices = self.api.plshell.GetSlices(self.api.plauth, slice_name) + #if slices: + #slice = slices[0] + #slivers = [] + #tags = self.api.plshell.GetSliceTags(self.api.plauth, slice['slice_tag_ids']) + #for node_id in slice['node_ids']: + #sliver = {} + #sliver['hostname'] = self.nodes[node_id]['hostname'] + #sliver['tags'] = [] + #slivers.append(sliver) + #for tag in tags: + ## if tag isn't bound to a node then it applies to all slivers + #if not tag['node_id']: + #sliver['tags'].append(tag) + #else: + #tag_host = self.nodes[tag['node_id']]['hostname'] + #if tag_host == sliver['hostname']: + #sliver.tags.append(tag) + #rspec.add_slivers(slivers) + return rspec.toxml() diff --git a/sfa/senslab/SenslabImport.py b/sfa/senslab/SenslabImport.py deleted file mode 120000 index b58f514c..00000000 --- a/sfa/senslab/SenslabImport.py +++ /dev/null @@ -1 +0,0 @@ -/home/avakian/Documents/sfa/sfa/senslab/SenslabImport.py \ No newline at end of file diff --git a/sfa/senslab/SenslabImport.py b/sfa/senslab/SenslabImport.py new file mode 100644 index 00000000..22fd5468 --- /dev/null +++ b/sfa/senslab/SenslabImport.py @@ -0,0 +1,251 @@ +# +# The import tool assumes that the existing PLC hierarchy should all be part +# of "planetlab.us" (see the root_auth and level1_auth variables below). +# +# Public keys are extracted from the users' SSH keys automatically and used to +# create GIDs. This is relatively experimental as a custom tool had to be +# written to perform conversion from SSH to OpenSSL format. It only supports +# RSA keys at this time, not DSA keys. +## + +import getopt +import sys +import tempfile + +from sfa.util.sfalogging import sfa_logger_goes_to_import,sfa_logger + +from sfa.util.record import * +from sfa.util.table import SfaTable +from sfa.util.xrn import get_authority, hrn_to_urn +from sfa.util.plxrn import email_to_hrn +from sfa.util.config import Config +from sfa.trust.certificate import convert_public_key, Keypair +from sfa.trust.trustedroot import * +from sfa.trust.hierarchy import * +from sfa.trust.gid import create_uuid + + +def _un_unicode(str): + if isinstance(str, unicode): + return str.encode("ascii", "ignore") + else: + return str + +def _cleanup_string(str): + # pgsql has a fit with strings that have high ascii in them, so filter it + # out when generating the hrns. + tmp = "" + for c in str: + if ord(c) < 128: + tmp = tmp + c + str = tmp + + str = _un_unicode(str) + str = str.replace(" ", "_") + str = str.replace(".", "_") + str = str.replace("(", "_") + str = str.replace("'", "_") + str = str.replace(")", "_") + str = str.replace('"', "_") + return str + +class SenslabImport: + + def __init__(self): + sfa_logger_goes_to_import() + self.logger = sfa_logger() + self.AuthHierarchy = Hierarchy() + self.config = Config() + self.TrustedRoots = TrustedRootList(Config.get_trustedroots_dir(self.config)) + print>>sys.stderr, "\r\n ========= \t\t SenslabImport TrustedRoots\r\n" , self.TrustedRoots + self.plc_auth = self.config.get_plc_auth() + self.root_auth = self.config.SFA_REGISTRY_ROOT_AUTH + + + def create_top_level_auth_records(self, hrn): + """ + Create top level records (includes root and sub authorities (local/remote) + """ + print>>sys.stderr, "\r\n =========SenslabImport create_top_level_auth_records\r\n" + urn = hrn_to_urn(hrn, 'authority') + # make sure parent exists + parent_hrn = get_authority(hrn) + if not parent_hrn: + parent_hrn = hrn + if not parent_hrn == hrn: + self.create_top_level_auth_records(parent_hrn) + + # create the authority if it doesnt already exist + if not self.AuthHierarchy.auth_exists(urn): + self.logger.info("Import: creating top level authorities") + self.AuthHierarchy.create_auth(urn) + + # create the db record if it doesnt already exist + auth_info = self.AuthHierarchy.get_auth_info(hrn) + table = SfaTable() + auth_record = table.find({'type': 'authority', 'hrn': hrn}) + + if not auth_record: + auth_record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=-1) + auth_record['authority'] = get_authority(auth_record['hrn']) + self.logger.info("Import: inserting authority record for %s"%hrn) + table.insert(auth_record) + print>>sys.stderr, "\r\n ========= \t\t SenslabImport NO AUTH RECORD \r\n" ,auth_record['authority'] + + + def create_interface_records(self): + """ + Create a record for each SFA interface + """ + # just create certs for all sfa interfaces even if they + # arent enabled + interface_hrn = self.config.SFA_INTERFACE_HRN + interfaces = ['authority+sa', 'authority+am', 'authority+sm'] + table = SfaTable() + auth_info = self.AuthHierarchy.get_auth_info(interface_hrn) + pkey = auth_info.get_pkey_object() + for interface in interfaces: + interface_record = table.find({'type': interface, 'hrn': interface_hrn}) + if not interface_record: + self.logger.info("Import: interface %s %s " % (interface_hrn, interface)) + urn = hrn_to_urn(interface_hrn, interface) + gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) + record = SfaRecord(hrn=interface_hrn, gid=gid, type=interface, pointer=-1) + record['authority'] = get_authority(interface_hrn) + print>>sys.stderr,"\r\n ==========create_interface_records", record['authority'] + table.insert(record) + + def import_person(self, parent_hrn, person, keys): + """ + Register a user record + """ + hrn = email_to_hrn(parent_hrn, person['email']) + + print >>sys.stderr , "\r\n_____00______SenslabImport : person", person + # ASN.1 will have problems with hrn's longer than 64 characters + if len(hrn) > 64: + hrn = hrn[:64] + print >>sys.stderr , "\r\n_____0______SenslabImport : parent_hrn", parent_hrn + self.logger.info("Import: person %s"%hrn) + key_ids = [] + # choper les cles ssh des users , sont ils dans OAR + if 'key_ids' in person and person['key_ids']: + key_ids = person["key_ids"] + # get the user's private key from the SSH keys they have uploaded + # to planetlab + + print >>sys.stderr , "\r\n_____1______SenslabImport : self.plc_auth %s \r\n \t keys %s key[0] %s" %(self.plc_auth,keys, keys[0]) + key = keys[0]['key'] + pkey = convert_public_key(key) + print >>sys.stderr , "\r\n_____2______SenslabImport : key %s pkey %s"% (key,pkey.as_pem()) + if not pkey: + pkey = Keypair(create=True) + else: + # the user has no keys + self.logger.warning("Import: person %s does not have a PL public key"%hrn) + # if a key is unavailable, then we still need to put something in the + # user's GID. So make one up. + pkey = Keypair(create=True) + print >>sys.stderr , "\r\n___ELSE________SenslabImport pkey : %s"%(pkey.key) + # create the gid + urn = hrn_to_urn(hrn, 'user') + print >>sys.stderr , "\r\n \t\t : urn ", urn + person_gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) + table = SfaTable() + person_record = SfaRecord(hrn=hrn, gid=person_gid, type="user", pointer=person['person_id']) + person_record['authority'] = get_authority(person_record['hrn']) + existing_records = table.find({'hrn': hrn, 'type': 'user', 'pointer': person['person_id']}) + if not existing_records: + table.insert(person_record) + else: + self.logger.info("Import: %s exists, updating " % hrn) + existing_record = existing_records[0] + person_record['record_id'] = existing_record['record_id'] + table.update(person_record) + + def import_slice(self, parent_hrn, slice): + slicename = slice['name'].split("_",1)[-1] + slicename = _cleanup_string(slicename) + + if not slicename: + self.logger.error("Import: failed to parse slice name %s" %slice['name']) + return + + hrn = parent_hrn + "." + slicename + self.logger.info("Import: slice %s"%hrn) + + pkey = Keypair(create=True) + urn = hrn_to_urn(hrn, 'slice') + slice_gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) + slice_record = SfaRecord(hrn=hrn, gid=slice_gid, type="slice", pointer=slice['slice_id']) + slice_record['authority'] = get_authority(slice_record['hrn']) + table = SfaTable() + existing_records = table.find({'hrn': hrn, 'type': 'slice', 'pointer': slice['slice_id']}) + if not existing_records: + table.insert(slice_record) + else: + self.logger.info("Import: %s exists, updating " % hrn) + existing_record = existing_records[0] + slice_record['record_id'] = existing_record['record_id'] + table.update(slice_record) + + def import_node(self, hrn, node): + self.logger.info("Import: node %s" % hrn) + # ASN.1 will have problems with hrn's longer than 64 characters + if len(hrn) > 64: + hrn = hrn[:64] + + table = SfaTable() + node_record = table.find({'type': 'node', 'hrn': hrn}) + pkey = Keypair(create=True) + urn = hrn_to_urn(hrn, 'node') + node_gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) + node_record = SfaRecord(hrn=hrn, gid=node_gid, type="node", pointer=node['node_id']) + node_record['authority'] = get_authority(node_record['hrn']) + existing_records = table.find({'hrn': hrn, 'type': 'node', 'pointer': node['node_id']}) + if not existing_records: + table.insert(node_record) + else: + self.logger.info("Import: %s exists, updating " % hrn) + existing_record = existing_records[0] + node_record['record_id'] = existing_record['record_id'] + table.update(node_record) + + + def import_site(self, parent_hrn, site): + plc_auth = self.plc_auth + sitename = site['login_base'] + sitename = _cleanup_string(sitename) + hrn = parent_hrn + "." + sitename + + urn = hrn_to_urn(hrn, 'authority') + self.logger.info("Import: site %s"%hrn) + + # create the authority + if not self.AuthHierarchy.auth_exists(urn): + self.AuthHierarchy.create_auth(urn) + + auth_info = self.AuthHierarchy.get_auth_info(urn) + + table = SfaTable() + auth_record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=site['site_id']) + auth_record['authority'] = get_authority(auth_record['hrn']) + existing_records = table.find({'hrn': hrn, 'type': 'authority', 'pointer': site['site_id']}) + if not existing_records: + table.insert(auth_record) + else: + self.logger.info("Import: %s exists, updating " % hrn) + existing_record = existing_records[0] + auth_record['record_id'] = existing_record['record_id'] + table.update(auth_record) + + return hrn + + + def delete_record(self, hrn, type): + # delete the record + table = SfaTable() + record_list = table.find({'type': type, 'hrn': hrn}) + for record in record_list: + self.logger.info("Import: removing record %s %s" % (type, hrn)) + table.remove(record) diff --git a/sfa/senslab/SenslabImportUsers.py b/sfa/senslab/SenslabImportUsers.py deleted file mode 120000 index 9880ae8c..00000000 --- a/sfa/senslab/SenslabImportUsers.py +++ /dev/null @@ -1 +0,0 @@ -/home/avakian/Documents/sfa/sfa/senslab/SenslabImportUsers.py \ No newline at end of file diff --git a/sfa/senslab/SenslabImportUsers.py b/sfa/senslab/SenslabImportUsers.py new file mode 100644 index 00000000..454a51d0 --- /dev/null +++ b/sfa/senslab/SenslabImportUsers.py @@ -0,0 +1,171 @@ +#!/usr/bin/python + +# import modules used here -- sys is a very standard one +import sys +import httplib +import json + + + + +def strip_dictionnary (dict_to_strip): + stripped_filter = [] + stripped_filterdict = {} + for f in dict_to_strip : + stripped_filter.append(str(f).strip('|')) + + stripped_filterdict = dict(zip(stripped_filter, dict_to_strip.values())) + + return stripped_filterdict + + +def filter_return_fields( dict_to_filter, return_fields): + filtered_dict = {} + #print>>sys.stderr, " \r\n \t \tfilter_return_fields return fields %s " %(return_fields) + for field in return_fields: + #print>>sys.stderr, " \r\n \t \tfield %s " %(field) + if field in dict_to_filter: + filtered_dict[field] = dict_to_filter[field] + #print>>sys.stderr, " \r\n \t\t filter_return_fields filtered_dict %s " %(filtered_dict) + return filtered_dict + + + +def parse_filter(list_to_filter, param_filter, type_of_list, return_fields=None) : + list_type = { 'persons': {'str': 'email','int':'person_id'}, 'keys':{'int':'key_id'}} + if type_of_list not in list_type: + print>>sys.stderr, " \r\n type_of_list Error parse_filter %s " %(type_of_list) + return [] + + print>>sys.stderr, " \r\n ____FIRST ENTRY parse_filter param_filter %s type %s " %(param_filter, type(param_filter)) + return_filtered_list= [] + + for item in list_to_filter: + tmp_item = {} + + if type(param_filter) is list : + #print>>sys.stderr, " \r\n p_filter LIST %s " %(param_filter) + + for p_filter in param_filter: + #print>>sys.stderr, " \r\n p_filter %s \t item %s " %(p_filter,item) + if type(p_filter) is int: + if item[list_type[type_of_list]['int']] == p_filter : + if return_fields: + tmp_item = filter_return_fields(item,return_fields) + else: + tmp_item = item + return_filtered_list.append(tmp_item) + #print>>sys.stderr, " \r\n 1tmp_item",tmp_item + + if type(p_filter) is str: + if item[list_type[type_of_list]['str']] == p_filter : + if return_fields: + tmp_item = filter_return_fields(item,return_fields) + else: + tmp_item = item + return_filtered_list.append(tmp_item) + #print>>sys.stderr, " \r\n 2tmp_item",tmp_item + + elif type(param_filter) is dict: + stripped_filterdict = strip_dictionnary(param_filter) + + tmp_copy = {} + tmp_copy = item.copy() + #print>>sys.stderr, " \r\n \t\t ________tmp_copy %s " %(tmp_copy) + key_list = tmp_copy.keys() + for key in key_list: + print>>sys.stderr, " \r\n \t\t key %s " %(key) + if key not in stripped_filterdict.keys(): + del tmp_copy[key] + + + print>>sys.stderr, " \r\n tmp_copy %s param_filter %s cmp = %s " %(tmp_copy, param_filter,cmp(tmp_copy, stripped_filterdict)) + + if cmp(tmp_copy, stripped_filterdict) == 0: + if return_fields: + tmp_item = filter_return_fields(item,return_fields) + else: + + tmp_item = item + return_filtered_list.append(tmp_item) + + return return_filtered_list + + +class SenslabImportUsers: + + + def __init__(self): + self.person_list = [] + self.keys_list = [] + #self.resources_fulldict['keys'] = [] + self.InitPersons() + self.InitKeys() + + def InitPersons(self): + persons_per_site = {} + person_id = 7 + persons_per_site[person_id] = {'person_id': person_id,'site_ids': [3],'email': 'a_rioot@senslab.fr', 'key_ids':[1], 'roles': ['pi'], 'role_ids':[20]} + person_id = 8 + persons_per_site[person_id] = {'person_id': person_id,'site_ids': [3],'email': 'lost@senslab.fr','key_ids':[1],'roles': ['pi'], 'role_ids':[20]} + for person_id in persons_per_site.keys(): + person = persons_per_site[person_id] + if person['person_id'] not in self.person_list: + self.person_list.append(person) + print>>sys.stderr, "InitPersons PERSON DICLIST", self.person_list + + + def InitKeys(self): + print>>sys.stderr, " InitKeys HEYYYYYYY\r\n" + + self.keys_list = [{'peer_key_id': None, 'key_type': 'ssh', 'key' :"ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEArcdW0X2la754SoFE+URbDsYP07AZJjrspMlvUc6u+4o6JpGRkqiv7XdkgOMIn6w3DF3cYCcA1Mc6XSG7gSD7eQx614cjlLmXzHpxSeidSs/LgZaAQpq9aQ0KhEiFxg0gp8TPeB5Z37YOPUumvcJr1ArwL/8tAOx3ClwgRhccr2HOe10YtZbMEboCarTlzNHiGolo7RYIJjGuG2RBSeAg6SMZrtnn0OdKBwp3iUlOfkS98eirVtWUp+G5+SZggip3fS3k5Oj7OPr1qauva8Rizt02Shz30DN9ikFNqV2KuPg54nC27/DQsQ6gtycARRVY91VvchmOk0HxFiW/9kS2GQ== root@FlabFedora2",'person_id': 7, 'key_id':1, 'peer_id':None }, {'peer_key_id': None, 'key_type': 'ssh', 'key' :"ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEArcdW0X2la754SoFE+URbDsYP07AZJjrspMlvUc6u+4o6JpGRkqiv7XdkgOMIn6w3DF3cYCcA1Mc6XSG7gSD7eQx614cjlLmXzHpxSeidSs/LgZaAQpq9aQ0KhEiFxg0gp8TPeB5Z37YOPUumvcJr1ArwL/8tAOx3ClwgRhccr2HOe10YtZbMEboCarTlzNHiGolo7RYIJjGuG2RBSeAg6SMZrtnn0OdKBwp3iUlOfkS98eirVtWUp+G5+SZggip3fS3k5Oj7OPr1qauva8Rizt02Shz30DN9ikFNqV2KuPg54nC27/DQsQ6gtycARRVY91VvchmOk0HxFiW/9kS2GQ== root@FlabFedora2",'person_id': 8, 'key_id':1, 'peer_id':None }] + + + + + def GetPersons(self, person_filter=None, return_fields=None): + print>>sys.stderr, " \r\n GetPersons person_filter %s return_fields %s" %(person_filter,return_fields) + if not self.person_list : + print>>sys.stderr, " \r\n ========>GetPersons NO PERSON LIST DAMMIT<========== \r\n" + + if not (person_filter or return_fields): + return self.person_list + + return_person_list= [] + return_person_list = parse_filter(self.person_list,person_filter ,'persons', return_fields) + return return_person_list + + + def GetKeys(self,key_filter=None, return_fields=None): + return_key_list= [] + print>>sys.stderr, " \r\n GetKeys" + + if not (key_filter or return_fields): + return self.keys_list + return_key_list = parse_filter(self.keys_list,key_filter ,'keys', return_fields) + return return_key_list + + #return_key_list= [] + #print>>sys.stderr, " \r\n GetKeys" + + #if not (key_filter or return_fields): + #return self.keys_list + + #elif key_filter or return_fields: + #for key in self.keys_list: + #tmp_key = {} + #if key_filter: + #for k_filter in key_filter: + #if key['key_id'] == k_filter : + #if return_fields: + #for field in return_fields: + #if field in key.keys(): + #tmp_key[field] = key[field] + #else: + #tmp_key = key + + #print>>sys.stderr, " \r\n tmp_key",tmp_key + #return_key_list.append(tmp_key) + #print>>sys.stderr," \r\n End GetKeys with filter ", return_key_list + #return return_key_list + \ No newline at end of file diff --git a/sfa/senslab/api.py b/sfa/senslab/api.py deleted file mode 120000 index 8d206633..00000000 --- a/sfa/senslab/api.py +++ /dev/null @@ -1 +0,0 @@ -/home/avakian/Documents/sfa/sfa/senslab/api.py \ No newline at end of file diff --git a/sfa/senslab/api.py b/sfa/senslab/api.py new file mode 100644 index 00000000..1df06ff8 --- /dev/null +++ b/sfa/senslab/api.py @@ -0,0 +1,708 @@ +# +# SFA XML-RPC and SOAP interfaces +# + +import sys +import os +import traceback +import string +import datetime +import xmlrpclib + +from sfa.util.faults import * +from sfa.util.api import * +from sfa.util.config import * +from sfa.util.sfalogging import sfa_logger +import sfa.util.xmlrpcprotocol as xmlrpcprotocol +from sfa.trust.auth import Auth +from sfa.trust.rights import Right, Rights, determine_rights +from sfa.trust.credential import Credential,Keypair +from sfa.trust.certificate import Certificate +from sfa.util.xrn import get_authority, hrn_to_urn +from sfa.util.plxrn import hostname_to_hrn, hrn_to_pl_slicename, hrn_to_pl_slicename, slicename_to_hrn +from sfa.util.nodemanager import NodeManager + +from sfa.senslab.OARrestapi import * +from sfa.senslab.SenslabImportUsers import * + +try: + from collections import defaultdict +except: + class defaultdict(dict): + def __init__(self, default_factory=None, *a, **kw): + if (default_factory is not None and + not hasattr(default_factory, '__call__')): + raise TypeError('first argument must be callable') + dict.__init__(self, *a, **kw) + self.default_factory = default_factory + def __getitem__(self, key): + try: + return dict.__getitem__(self, key) + except KeyError: + return self.__missing__(key) + def __missing__(self, key): + if self.default_factory is None: + raise KeyError(key) + self[key] = value = self.default_factory() + return value + def __reduce__(self): + if self.default_factory is None: + args = tuple() + else: + args = self.default_factory, + return type(self), args, None, None, self.items() + def copy(self): + return self.__copy__() + def __copy__(self): + return type(self)(self.default_factory, self) + def __deepcopy__(self, memo): + import copy + return type(self)(self.default_factory, + copy.deepcopy(self.items())) + def __repr__(self): + return 'defaultdict(%s, %s)' % (self.default_factory, + dict.__repr__(self)) +## end of http://code.activestate.com/recipes/523034/ }}} + +def list_to_dict(recs, key): + """ + convert a list of dictionaries into a dictionary keyed on the + specified dictionary key + """ + # print>>sys.stderr, " \r\n \t\t 1list_to_dict : rec %s \r\n \t\t list_to_dict key %s" %(recs,key) + keys = [rec[key] for rec in recs] + print>>sys.stderr, " \r\n \t\t list_to_dict : rec %s \r\n \t\t list_to_dict keys %s" %(recs,keys) + return dict(zip(keys, recs)) + +class SfaAPI(BaseAPI): + + # flat list of method names + import sfa.methods + methods = sfa.methods.all + + def __init__(self, config = "/etc/sfa/sfa_config.py", encoding = "utf-8", + methods='sfa.methods', peer_cert = None, interface = None, + key_file = None, cert_file = None, cache = None): + BaseAPI.__init__(self, config=config, encoding=encoding, methods=methods, \ + peer_cert=peer_cert, interface=interface, key_file=key_file, \ + cert_file=cert_file, cache=cache) + + self.encoding = encoding + from sfa.util.table import SfaTable + self.SfaTable = SfaTable + # Better just be documenting the API + if config is None: + return + print >>sys.stderr, "\r\n_____________ SFA SENSLAB API \r\n" + # Load configuration + self.config = Config(config) + self.auth = Auth(peer_cert) + self.interface = interface + self.key_file = key_file + self.key = Keypair(filename=self.key_file) + self.cert_file = cert_file + self.cert = Certificate(filename=self.cert_file) + self.credential = None + # Initialize the PLC shell only if SFA wraps a myPLC + rspec_type = self.config.get_aggregate_type() + self.oar = OARapi() + self.users = SenslabImportUsers() + self.hrn = self.config.SFA_INTERFACE_HRN + self.time_format = "%Y-%m-%d %H:%M:%S" + self.logger=sfa_logger() + print >>sys.stderr, "\r\n \t\t___________PLC/API.PY __init__ STOP ",self.interface #dir(self) + + + #def getPLCShell(self): + #self.plauth = {'Username': self.config.SFA_PLC_USER, + #'AuthMethod': 'password', + #'AuthString': self.config.SFA_PLC_PASSWORD} + #try: + #print >>sys.stderr, "\r\n \t\t___________PLC/API.PY getPLCShell " + #sys.path.append(os.path.dirname(os.path.realpath("/usr/bin/plcsh"))) + #self.plshell_type = 'direct' + #import PLC.Shell + #shell = PLC.Shell.Shell(globals = globals()) + #print >>sys.stderr, "\r\n \t\t____tryshell %s \r\n type %s \r\n %s"%(shell, type(shell), dir(shell)) + + #except: + #self.plshell_type = 'xmlrpc' + #print >>sys.stderr, "\r\n \t\t___________PLC/API.PY getPLCShell xmlrpc" + #url = self.config.SFA_PLC_URL + #shell = xmlrpclib.Server(url, verbose = 0, allow_none = True) + + #return shell + + def getCredential(self): + """ + Return a valid credential for this interface. + """ + type = 'authority' + path = self.config.SFA_DATA_DIR + filename = ".".join([self.interface, self.hrn, type, "cred"]) + cred_filename = path + os.sep + filename + cred = None + if os.path.isfile(cred_filename): + cred = Credential(filename = cred_filename) + # make sure cred isnt expired + if not cred.get_expiration or \ + datetime.datetime.today() < cred.get_expiration(): + return cred.save_to_string(save_parents=True) + + # get a new credential + if self.interface in ['registry']: + cred = self.__getCredentialRaw() + else: + cred = self.__getCredential() + cred.save_to_file(cred_filename, save_parents=True) + + return cred.save_to_string(save_parents=True) + + + def getDelegatedCredential(self, creds): + """ + Attempt to find a credential delegated to us in + the specified list of creds. + """ + if creds and not isinstance(creds, list): + creds = [creds] + delegated_creds = filter_creds_by_caller(creds,self.hrn) + if not delegated_creds: + return None + return delegated_creds[0] + + def __getCredential(self): + """ + Get our credential from a remote registry + """ + from sfa.server.registry import Registries + registries = Registries(self) + registry = registries[self.hrn] + cert_string=self.cert.save_to_string(save_parents=True) + # get self credential + self_cred = registry.GetSelfCredential(cert_string, self.hrn, 'authority') + # get credential + cred = registry.GetCredential(self_cred, self.hrn, 'authority') + return Credential(string=cred) + + def __getCredentialRaw(self): + """ + Get our current credential directly from the local registry. + """ + + hrn = self.hrn + auth_hrn = self.auth.get_authority(hrn) + + # is this a root or sub authority + if not auth_hrn or hrn == self.config.SFA_INTERFACE_HRN: + auth_hrn = hrn + auth_info = self.auth.get_auth_info(auth_hrn) + table = self.SfaTable() + records = table.findObjects(hrn) + if not records: + raise RecordNotFound + record = records[0] + type = record['type'] + object_gid = record.get_gid_object() + new_cred = Credential(subject = object_gid.get_subject()) + new_cred.set_gid_caller(object_gid) + new_cred.set_gid_object(object_gid) + new_cred.set_issuer_keys(auth_info.get_privkey_filename(), auth_info.get_gid_filename()) + + r1 = determine_rights(type, hrn) + new_cred.set_privileges(r1) + new_cred.encode() + new_cred.sign() + + return new_cred + + + def loadCredential (self): + """ + Attempt to load credential from file if it exists. If it doesnt get + credential from registry. + """ + + # see if this file exists + # XX This is really the aggregate's credential. Using this is easier than getting + # the registry's credential from iteslf (ssl errors). + ma_cred_filename = self.config.SFA_DATA_DIR + os.sep + self.interface + self.hrn + ".ma.cred" + try: + self.credential = Credential(filename = ma_cred_filename) + except IOError: + self.credential = self.getCredentialFromRegistry() + + ## + # Convert SFA fields to PLC fields for use when registering up updating + # registry record in the PLC database + # + # @param type type of record (user, slice, ...) + # @param hrn human readable name + # @param sfa_fields dictionary of SFA fields + # @param pl_fields dictionary of PLC fields (output) + + def sfa_fields_to_pl_fields(self, type, hrn, record): + + def convert_ints(tmpdict, int_fields): + for field in int_fields: + if field in tmpdict: + tmpdict[field] = int(tmpdict[field]) + + pl_record = {} + #for field in record: + # pl_record[field] = record[field] + + if type == "slice": + if not "instantiation" in pl_record: + pl_record["instantiation"] = "plc-instantiated" + pl_record["name"] = hrn_to_pl_slicename(hrn) + if "url" in record: + pl_record["url"] = record["url"] + if "description" in record: + pl_record["description"] = record["description"] + if "expires" in record: + pl_record["expires"] = int(record["expires"]) + + elif type == "node": + if not "hostname" in pl_record: + if not "hostname" in record: + raise MissingSfaInfo("hostname") + pl_record["hostname"] = record["hostname"] + if not "model" in pl_record: + pl_record["model"] = "geni" + + elif type == "authority": + pl_record["login_base"] = hrn_to_pl_login_base(hrn) + + if not "name" in pl_record: + pl_record["name"] = hrn + + if not "abbreviated_name" in pl_record: + pl_record["abbreviated_name"] = hrn + + if not "enabled" in pl_record: + pl_record["enabled"] = True + + if not "is_public" in pl_record: + pl_record["is_public"] = True + + return pl_record + + def fill_record_pl_info(self, records): + """ + Fill in the planetlab specific fields of a SFA record. This + involves calling the appropriate PLC method to retrieve the + database record for the object. + + PLC data is filled into the pl_info field of the record. + + @param record: record to fill in field (in/out param) + """ + # get ids by type + print>>sys.stderr, "\r\n \t\t \t fill_record_pl_info : entering... \r\n" + node_ids, site_ids, slice_ids = [], [], [] + person_ids, key_ids = [], [] + type_map = {'node': node_ids, 'authority': site_ids, + 'slice': slice_ids, 'user': person_ids} + + for record in records: + for type in type_map: + print>>sys.stderr, "\r\n \t\t \t fill_record_pl_info : type %s. record['pointer'] %s "%(type,record['pointer']) + if type == record['type']: + type_map[type].append(record['pointer']) + print>>sys.stderr, "\r\n \t\t \t fill_record_pl_info : records %s... \r\n \t\t \t fill_record_pl_info : type_map %s"%(records,type_map) + # get pl records + nodes, sites, slices, persons, keys = {}, {}, {}, {}, {} + if node_ids: + node_list = self.oar.GetNodes( node_ids) + print>>sys.stderr, " \r\n \t\t\t BEFORE LIST_TO_DICT_NODES node_ids : %s" %(node_ids) + nodes = list_to_dict(node_list, 'node_id') + if site_ids: + site_list = self.oar.GetSites( site_ids) + sites = list_to_dict(site_list, 'site_id') + if slice_ids: + slice_list = self.oar.GetSlices( slice_ids) + slices = list_to_dict(slice_list, 'slice_id') + if person_ids: + print>>sys.stderr, " \r\n \t\t \t fill_record_pl_info BEFORE GetPersons person_ids: %s" %(person_ids) + person_list = self.users.GetPersons( person_ids) + persons = list_to_dict(person_list, 'person_id') + print>>sys.stderr, "\r\n fill_record_pl_info persons %s \r\n \t\t person_ids %s " %(persons, person_ids) + for person in persons: + key_ids.extend(persons[person]['key_ids']) + print>>sys.stderr, "\r\n key_ids %s " %(key_ids) + + pl_records = {'node': nodes, 'authority': sites, + 'slice': slices, 'user': persons} + + if key_ids: + key_list = self.users.GetKeys( key_ids) + keys = list_to_dict(key_list, 'key_id') + # print>>sys.stderr, "\r\n fill_record_pl_info persons %s \r\n \t\t keys %s " %(keys) + # fill record info + for record in records: + # records with pointer==-1 do not have plc info. + # for example, the top level authority records which are + # authorities, but not PL "sites" + if record['pointer'] == -1: + continue + + for type in pl_records: + if record['type'] == type: + if record['pointer'] in pl_records[type]: + record.update(pl_records[type][record['pointer']]) + break + # fill in key info + if record['type'] == 'user': + if 'key_ids' not in record: + print>>sys.stderr, " NO_KEY_IDS fill_record_pl_info key_ids record: %s" %(record) + else: + pubkeys = [keys[key_id]['key'] for key_id in record['key_ids'] if key_id in keys] + record['keys'] = pubkeys + + # fill in record hrns + records = self.fill_record_hrns(records) + + return records + + def fill_record_hrns(self, records): + """ + convert pl ids to hrns + """ + + # get ids + slice_ids, person_ids, site_ids, node_ids = [], [], [], [] + for record in records: + if 'site_id' in record: + site_ids.append(record['site_id']) + if 'site_ids' in records: + site_ids.extend(record['site_ids']) + if 'person_ids' in record: + person_ids.extend(record['person_ids']) + if 'slice_ids' in record: + slice_ids.extend(record['slice_ids']) + if 'node_ids' in record: + node_ids.extend(record['node_ids']) + + # get pl records + slices, persons, sites, nodes = {}, {}, {}, {} + if site_ids: + site_list = self.oar.GetSites( site_ids, ['site_id', 'login_base']) + sites = list_to_dict(site_list, 'site_id') + print>>sys.stderr, " \r\n \r\n \t\t ____ site_list %s \r\n \t\t____ sites %s " % (site_list,sites) + if person_ids: + person_list = self.users.GetPersons( person_ids, ['person_id', 'email']) + persons = list_to_dict(person_list, 'person_id') + if slice_ids: + slice_list = self.oar.GetSlices( slice_ids, ['slice_id', 'name']) + slices = list_to_dict(slice_list, 'slice_id') + if node_ids: + node_list = self.oar.GetNodes( node_ids, ['node_id', 'hostname']) + nodes = list_to_dict(node_list, 'node_id') + + # convert ids to hrns + for record in records: + + # get all relevant data + type = record['type'] + pointer = record['pointer'] + auth_hrn = self.hrn + login_base = '' + if pointer == -1: + continue + + print>>sys.stderr, " \r\n \r\n \t\t fill_record_hrns : sites %s \r\n \t\t record %s " %(sites, record) + if 'site_id' in record: + site = sites[record['site_id']] + print>>sys.stderr, " \r\n \r\n \t\t \t fill_record_hrns : sites %s \r\n \t\t\t site sites[record['site_id']] %s " %(sites,site) + login_base = site['login_base'] + record['site'] = ".".join([auth_hrn, login_base]) + if 'person_ids' in record: + emails = [persons[person_id]['email'] for person_id in record['person_ids'] \ + if person_id in persons] + usernames = [email.split('@')[0] for email in emails] + person_hrns = [".".join([auth_hrn, login_base, username]) for username in usernames] + record['persons'] = person_hrns + if 'slice_ids' in record: + slicenames = [slices[slice_id]['name'] for slice_id in record['slice_ids'] \ + if slice_id in slices] + slice_hrns = [slicename_to_hrn(auth_hrn, slicename) for slicename in slicenames] + record['slices'] = slice_hrns + if 'node_ids' in record: + hostnames = [nodes[node_id]['hostname'] for node_id in record['node_ids'] \ + if node_id in nodes] + node_hrns = [hostname_to_hrn(auth_hrn, login_base, hostname) for hostname in hostnames] + record['nodes'] = node_hrns + if 'site_ids' in record: + login_bases = [sites[site_id]['login_base'] for site_id in record['site_ids'] \ + if site_id in sites] + site_hrns = [".".join([auth_hrn, lbase]) for lbase in login_bases] + record['sites'] = site_hrns + + return records + + def fill_record_sfa_info(self, records): + + def startswith(prefix, values): + return [value for value in values if value.startswith(prefix)] + + SenslabUsers = SenslabImportUsers() + # get person ids + person_ids = [] + site_ids = [] + for record in records: + person_ids.extend(record.get("person_ids", [])) + site_ids.extend(record.get("site_ids", [])) + if 'site_id' in record: + site_ids.append(record['site_id']) + + print>>sys.stderr, "\r\n \r\n _fill_record_sfa_info ___person_ids %s \r\n \t\t site_ids %s " %(person_ids, site_ids) + + # get all pis from the sites we've encountered + # and store them in a dictionary keyed on site_id + site_pis = {} + if site_ids: + pi_filter = {'|roles': ['pi'], '|site_ids': site_ids} + pi_list = SenslabUsers.GetPersons( pi_filter, ['person_id', 'site_ids']) + print>>sys.stderr, "\r\n \r\n _fill_record_sfa_info ___ GetPersons ['person_id', 'site_ids'] pi_ilist %s" %(pi_list) + + for pi in pi_list: + # we will need the pi's hrns also + person_ids.append(pi['person_id']) + + # we also need to keep track of the sites these pis + # belong to + for site_id in pi['site_ids']: + if site_id in site_pis: + site_pis[site_id].append(pi) + else: + site_pis[site_id] = [pi] + + # get sfa records for all records associated with these records. + # we'll replace pl ids (person_ids) with hrns from the sfa records + # we obtain + + # get the sfa records + table = self.SfaTable() + person_list, persons = [], {} + person_list = table.find({'type': 'user', 'pointer': person_ids}) + # create a hrns keyed on the sfa record's pointer. + # Its possible for multiple records to have the same pointer so + # the dict's value will be a list of hrns. + persons = defaultdict(list) + for person in person_list: + persons[person['pointer']].append(person) + + # get the pl records + pl_person_list, pl_persons = [], {} + pl_person_list = SenslabUsers.GetPersons(person_ids, ['person_id', 'roles']) + pl_persons = list_to_dict(pl_person_list, 'person_id') + print>>sys.stderr, "\r\n \r\n _fill_record_sfa_info ___ _list %s \r\n \t\tplshell.GetPersons ['person_id', 'roles'] pl_persons %s " %(pl_person_list, pl_persons) + # fill sfa info + for record in records: + # skip records with no pl info (top level authorities) + if record['pointer'] == -1: + continue + sfa_info = {} + type = record['type'] + if (type == "slice"): + # all slice users are researchers + record['PI'] = [] + record['researcher'] = [] + for person_id in record['person_ids']: + hrns = [person['hrn'] for person in persons[person_id]] + record['researcher'].extend(hrns) + + # pis at the slice's site + pl_pis = site_pis[record['site_id']] + pi_ids = [pi['person_id'] for pi in pl_pis] + for person_id in pi_ids: + hrns = [person['hrn'] for person in persons[person_id]] + record['PI'].extend(hrns) + record['geni_urn'] = hrn_to_urn(record['hrn'], 'slice') + record['geni_creator'] = record['PI'] + + elif (type == "authority"): + record['PI'] = [] + record['operator'] = [] + record['owner'] = [] + for pointer in record['person_ids']: + if pointer not in persons or pointer not in pl_persons: + # this means there is not sfa or pl record for this user + continue + hrns = [person['hrn'] for person in persons[pointer]] + roles = pl_persons[pointer]['roles'] + if 'pi' in roles: + record['PI'].extend(hrns) + if 'tech' in roles: + record['operator'].extend(hrns) + if 'admin' in roles: + record['owner'].extend(hrns) + # xxx TODO: OrganizationName + elif (type == "node"): + sfa_info['dns'] = record.get("hostname", "") + # xxx TODO: URI, LatLong, IP, DNS + + elif (type == "user"): + sfa_info['email'] = record.get("email", "") + sfa_info['geni_urn'] = hrn_to_urn(record['hrn'], 'user') + sfa_info['geni_certificate'] = record['gid'] + # xxx TODO: PostalAddress, Phone + record.update(sfa_info) + + def fill_record_info(self, records): + """ + Given a SFA record, fill in the PLC specific and SFA specific + fields in the record. + """ + print >>sys.stderr, "\r\n \t\t fill_record_info %s"%(records) + if not isinstance(records, list): + records = [records] + print >>sys.stderr, "\r\n \t\t BEFORE fill_record_pl_info %s" %(records) + self.fill_record_pl_info(records) + print >>sys.stderr, "\r\n \t\t after fill_record_pl_info %s" %(records) + self.fill_record_sfa_info(records) + print >>sys.stderr, "\r\n \t\t after fill_record_sfa_info" + + def update_membership_list(self, oldRecord, record, listName, addFunc, delFunc): + # get a list of the HRNs tht are members of the old and new records + if oldRecord: + oldList = oldRecord.get(listName, []) + else: + oldList = [] + newList = record.get(listName, []) + + # if the lists are the same, then we don't have to update anything + if (oldList == newList): + return + + # build a list of the new person ids, by looking up each person to get + # their pointer + newIdList = [] + table = self.SfaTable() + records = table.find({'type': 'user', 'hrn': newList}) + for rec in records: + newIdList.append(rec['pointer']) + + # build a list of the old person ids from the person_ids field + if oldRecord: + oldIdList = oldRecord.get("person_ids", []) + containerId = oldRecord.get_pointer() + else: + # if oldRecord==None, then we are doing a Register, instead of an + # update. + oldIdList = [] + containerId = record.get_pointer() + + # add people who are in the new list, but not the oldList + for personId in newIdList: + if not (personId in oldIdList): + addFunc(self.plauth, personId, containerId) + + # remove people who are in the old list, but not the new list + for personId in oldIdList: + if not (personId in newIdList): + delFunc(self.plauth, personId, containerId) + + def update_membership(self, oldRecord, record): + if record.type == "slice": + self.update_membership_list(oldRecord, record, 'researcher', + self.plshell.AddPersonToSlice, + self.plshell.DeletePersonFromSlice) + elif record.type == "authority": + # xxx TODO + pass + + + +class ComponentAPI(BaseAPI): + + def __init__(self, config = "/etc/sfa/sfa_config.py", encoding = "utf-8", methods='sfa.methods', + peer_cert = None, interface = None, key_file = None, cert_file = None): + + BaseAPI.__init__(self, config=config, encoding=encoding, methods=methods, peer_cert=peer_cert, + interface=interface, key_file=key_file, cert_file=cert_file) + self.encoding = encoding + + # Better just be documenting the API + if config is None: + return + + self.nodemanager = NodeManager(self.config) + + def sliver_exists(self): + sliver_dict = self.nodemanager.GetXIDs() + if slicename in sliver_dict.keys(): + return True + else: + return False + + def get_registry(self): + addr, port = self.config.SFA_REGISTRY_HOST, self.config.SFA_REGISTRY_PORT + url = "http://%(addr)s:%(port)s" % locals() + server = xmlrpcprotocol.get_server(url, self.key_file, self.cert_file) + return server + + def get_node_key(self): + # this call requires no authentication, + # so we can generate a random keypair here + subject="component" + (kfd, keyfile) = tempfile.mkstemp() + (cfd, certfile) = tempfile.mkstemp() + key = Keypair(create=True) + key.save_to_file(keyfile) + cert = Certificate(subject=subject) + cert.set_issuer(key=key, subject=subject) + cert.set_pubkey(key) + cert.sign() + cert.save_to_file(certfile) + registry = self.get_registry() + # the registry will scp the key onto the node + registry.get_key() + + def getCredential(self): + """ + Get our credential from a remote registry + """ + path = self.config.SFA_DATA_DIR + config_dir = self.config.config_path + cred_filename = path + os.sep + 'node.cred' + try: + credential = Credential(filename = cred_filename) + return credential.save_to_string(save_parents=True) + except IOError: + node_pkey_file = config_dir + os.sep + "node.key" + node_gid_file = config_dir + os.sep + "node.gid" + cert_filename = path + os.sep + 'server.cert' + if not os.path.exists(node_pkey_file) or \ + not os.path.exists(node_gid_file): + self.get_node_key() + + # get node's hrn + gid = GID(filename=node_gid_file) + hrn = gid.get_hrn() + # get credential from registry + cert_str = Certificate(filename=cert_filename).save_to_string(save_parents=True) + registry = self.get_registry() + cred = registry.GetSelfCredential(cert_str, hrn, 'node') + Credential(string=cred).save_to_file(credfile, save_parents=True) + + return cred + + def clean_key_cred(self): + """ + remove the existing keypair and cred and generate new ones + """ + files = ["server.key", "server.cert", "node.cred"] + for f in files: + filepath = KEYDIR + os.sep + f + if os.path.isfile(filepath): + os.unlink(f) + + # install the new key pair + # GetCredential will take care of generating the new keypair + # and credential + self.get_node_key() + self.getCredential() + + diff --git a/sfa/senslab/senslab-import.py b/sfa/senslab/senslab-import.py deleted file mode 120000 index 0ea84d87..00000000 --- a/sfa/senslab/senslab-import.py +++ /dev/null @@ -1 +0,0 @@ -/home/avakian/Documents/sfa/sfa/senslab/senslab-import.py \ No newline at end of file diff --git a/sfa/senslab/senslab-import.py b/sfa/senslab/senslab-import.py new file mode 100755 index 00000000..735cc10e --- /dev/null +++ b/sfa/senslab/senslab-import.py @@ -0,0 +1,233 @@ +#!/usr/bin/python +# +## +# Import PLC records into the SFA database. It is indended that this tool be +# run once to create SFA records that reflect the current state of the +# planetlab database. +# +# The import tool assumes that the existing PLC hierarchy should all be part +# of "planetlab.us" (see the root_auth and level1_auth variables below). +# +# Public keys are extracted from the users' SSH keys automatically and used to +# create GIDs. This is relatively experimental as a custom tool had to be +# written to perform conversion from SSH to OpenSSL format. It only supports +# RSA keys at this time, not DSA keys. +## + +import getopt +import sys +import tempfile + + +from sfa.util.record import * +from sfa.util.table import SfaTable +from sfa.util.xrn import get_leaf, get_authority +from sfa.util.plxrn import hostname_to_hrn, slicename_to_hrn, email_to_hrn, hrn_to_pl_slicename +from sfa.util.config import Config +from sfa.trust.certificate import convert_public_key, Keypair +from sfa.trust.trustedroot import * +from sfa.trust.hierarchy import * +from sfa.util.xrn import Xrn +from sfa.trust.gid import create_uuid + + +from sfa.senslab.SenslabImportUsers import * +from sfa.senslab.OARrestapi import * + +from sfa.senslab.SenslabImport import SenslabImport + + + + + +oarserver = {} +oarserver['ip'] = '10.127.255.254' +oarserver['port'] = 80 +oarserver['uri'] = '/oarapi/resources/full.json' + + +def process_options(): + + (options, args) = getopt.getopt(sys.argv[1:], '', []) + for opt in options: + name = opt[0] + val = opt[1] + + +def load_keys(filename): + keys = {} + tmp_dict = {} + try: + execfile(filename, tmp_dict) + if 'keys' in tmp_dict: + keys = tmp_dict['keys'] + return keys + except: + return keys + +def save_keys(filename, keys): + f = open(filename, 'w') + f.write("keys = %s" % str(keys)) + f.close() + +def main(): + + process_options() + config = Config() + if not config.SFA_REGISTRY_ENABLED: + sys.exit(0) + root_auth = config.SFA_REGISTRY_ROOT_AUTH + interface_hrn = config.SFA_INTERFACE_HRN + print interface_hrn, root_auth + keys_filename = config.config_path + os.sep + 'person_keys.py' + + sfaImporter = SenslabImport() + SenslabUsers = SenslabImportUsers() + + OARImporter = OARapi() + #print '\r\n =====OAR Importer list===== ' + #for node in OARImporter.OARserver.GetNodes().keys(): + #print node, OARImporter.OARserver.GetNodes[node] + + + #if config.SFA_API_DEBUG: sfaImporter.logger.setLevelDebug() + #shell = sfaImporter.shell + #plc_auth = sfaImporter.plc_auth + #print plc_auth + + # initialize registry db table + table = SfaTable() + if not table.exists(): + table.create() + + # create root authority + sfaImporter.create_top_level_auth_records(root_auth) + if not root_auth == interface_hrn: + sfaImporter.create_top_level_auth_records(interface_hrn) + + # create interface records ADDED 12 JUILLET 2011 + sfaImporter.logger.info("Import: creating interface records") + sfaImporter.create_interface_records() + + # add local root authority's cert to trusted list ADDED 12 JUILLET 2011 + sfaImporter.logger.info("Import: adding " + interface_hrn + " to trusted list") + authority = sfaImporter.AuthHierarchy.get_auth_info(interface_hrn) + sfaImporter.TrustedRoots.add_gid(authority.get_gid_object()) + + + print "\r\n \r\n create dict of all existing sfa records" + # create dict of all existing sfa records + existing_records = {} + existing_hrns = [] + key_ids = [] + person_keys = {} + results = table.find() + for result in results: + existing_records[(result['hrn'], result['type'])] = result + existing_hrns.append(result['hrn']) + + # Get all Senslab sites + sites_dict = OARImporter.GetSites() + print "\r\n sSITES_DICT" , sites_dict + + + #Get Senslab nodes + nodes_dict = OARImporter.GetNodes() + print "\r\n NODES8DICT ",nodes_dict + + persons_list = SenslabUsers.GetPersons() + print "\r\n PERSONS_LIST ",persons_list + + keys_list = SenslabUsers.GetKeys() + print "\r\n KEYSS_LIST ",keys_list + + # start importing + for site in sites_dict: + site_hrn = interface_hrn + "." + site['login_base'] + #sfa_logger().info("Importing site: %s" % site_hrn) + print "HRN %s %s site existing in hrn ? %s" %( site['login_base'],site_hrn, site_hrn in existing_hrns) + # import if hrn is not in list of existing hrns or if the hrn exists + # but its not a site record + if site_hrn not in existing_hrns or \ + (site_hrn, 'authority') not in existing_records: + print "SITE HRN UNKNOWN" , site, site_hrn + site_hrn = sfaImporter.import_site(interface_hrn, site) + + print "\r\n \r\n ===========IMPORT NODE8RECORDS ==========\r\n site %s \r\n \t nodes_dict %s" %(site,nodes_dict) + # import node records + for node_id in site['node_ids']: + #for[node['node_id'] for node in nodes_dict]: + #print '\r\n \t **NODE_ID %s node %s '%( node_id, node) + #continue + for node in nodes_dict: + if node_id is node['node_id']: + #node = nodes_dict[node_id] + print '\r\n \t NODE_ID %s node %s '%( node_id, node) + hrn = hostname_to_hrn(interface_hrn, site['login_base'], node['hostname']) + break + + if hrn not in existing_hrns or \ + (hrn, 'node') not in existing_records: + print "\t\t NODE HRN NOT in existing records!" ,hrn + sfaImporter.import_node(hrn, node) + + + for person in persons_list: + hrn = email_to_hrn(site_hrn, person['email']) + print >>sys.stderr, "\r\n\r\n^^^^^^^^^^^^^PERSON hrn %s person %s site hrn %s" %(hrn,person,site_hrn) + sfaImporter.import_person( site_hrn, person,keys_list) + + # remove stale records + for (record_hrn, type) in existing_records.keys(): + record = existing_records[(record_hrn, type)] + print" \r\n ****record hrn %s \t\t TYPE %s " %(record_hrn,type) + # if this is the interface name dont do anything + if record_hrn == interface_hrn or \ + record_hrn == root_auth or \ + record['peer_authority']: + continue + + + found = False + + if type == 'authority': + for site in sites_dict: + print "\t type : authority : ", site + site_hrn = interface_hrn + "." + site['login_base'] + if site_hrn == record_hrn and site['site_id'] == record['pointer']: + found = True + print "\t \t Found :", found + break + + elif type == 'node': + login_base = get_leaf(get_authority(record_hrn)) + + nodename = Xrn.unescape(get_leaf(record_hrn)) + print "type: node : login_base %s nodename %s" %(login_base, nodename) + if login_base in sites_dict: + site = sites_dict[login_base] + print "\t type node : login base %s site %s" %(login_base, site) + for node in nodes_dict.values(): + tmp_nodename = node['hostname'] + if tmp_nodename == nodename and \ + node['site_id'] == site['site_id'] and \ + node['node_id'] == record['pointer']: + found = True + print "\t Nodename: %s site id %s node id %s record %s" %( nodename, node['site_id'], node['node_id'],record['pointer']) + break + else: + continue + + if not found: + record_object = existing_records[(record_hrn, type)] + print "\t\t NOT FOUND ! " + sfaImporter.delete_record(record_hrn, type) + + # save pub keys + sfaImporter.logger.info('Import: saving current pub keys') + save_keys(keys_filename, person_keys) + + + +if __name__ == "__main__": + main() diff --git a/sfa/senslab/sfaImport.py b/sfa/senslab/sfaImport.py deleted file mode 120000 index 6e88ef31..00000000 --- a/sfa/senslab/sfaImport.py +++ /dev/null @@ -1 +0,0 @@ -/home/avakian/Documents/sfa/sfa/senslab/sfaImport.py \ No newline at end of file diff --git a/sfa/senslab/sfaImport.py b/sfa/senslab/sfaImport.py new file mode 100644 index 00000000..3f85f4bb --- /dev/null +++ b/sfa/senslab/sfaImport.py @@ -0,0 +1,272 @@ +# +# The import tool assumes that the existing PLC hierarchy should all be part +# of "planetlab.us" (see the root_auth and level1_auth variables below). +# +# Public keys are extracted from the users' SSH keys automatically and used to +# create GIDs. This is relatively experimental as a custom tool had to be +# written to perform conversion from SSH to OpenSSL format. It only supports +# RSA keys at this time, not DSA keys. +## + +import getopt +import sys +import tempfile + +from sfa.util.sfalogging import sfa_logger_goes_to_import,sfa_logger + +from sfa.util.record import * +from sfa.util.table import SfaTable +from sfa.util.xrn import get_authority, hrn_to_urn +from sfa.util.plxrn import email_to_hrn +from sfa.util.config import Config +from sfa.trust.certificate import convert_public_key, Keypair +from sfa.trust.trustedroot import * +from sfa.trust.hierarchy import * +from sfa.trust.gid import create_uuid + + +def _un_unicode(str): + if isinstance(str, unicode): + return str.encode("ascii", "ignore") + else: + return str + +def _cleanup_string(str): + # pgsql has a fit with strings that have high ascii in them, so filter it + # out when generating the hrns. + tmp = "" + for c in str: + if ord(c) < 128: + tmp = tmp + c + str = tmp + + str = _un_unicode(str) + str = str.replace(" ", "_") + str = str.replace(".", "_") + str = str.replace("(", "_") + str = str.replace("'", "_") + str = str.replace(")", "_") + str = str.replace('"', "_") + return str + +class sfaImport: + + def __init__(self): + sfa_logger_goes_to_import() + self.logger = sfa_logger() + self.AuthHierarchy = Hierarchy() + self.config = Config() + self.TrustedRoots = TrustedRootList(Config.get_trustedroots_dir(self.config)) + + self.plc_auth = self.config.get_plc_auth() + self.root_auth = self.config.SFA_REGISTRY_ROOT_AUTH + print>>sys.stderr, "\r\n ========= \t\t sfaImport plc_auth %s root_auth %s \r\n" %( self.plc_auth, self.root_auth ) + # connect to planetlab + self.shell = None + if "Url" in self.plc_auth: + from sfa.plc.remoteshell import RemoteShell + self.shell = RemoteShell(self.logger) + else: + import PLC.Shell + self.shell = PLC.Shell.Shell(globals = globals()) + + def create_top_level_auth_records(self, hrn): + """ + Create top level records (includes root and sub authorities (local/remote) + """ + + urn = hrn_to_urn(hrn, 'authority') + # make sure parent exists + parent_hrn = get_authority(hrn) + if not parent_hrn: + parent_hrn = hrn + if not parent_hrn == hrn: + self.create_top_level_auth_records(parent_hrn) + print>>sys.stderr, "\r\n =========create_top_level_auth_records parent_hrn \r\n", parent_hrn + + # create the authority if it doesnt already exist + if not self.AuthHierarchy.auth_exists(urn): + self.logger.info("Import: creating top level authorities") + self.AuthHierarchy.create_auth(urn) + + # create the db record if it doesnt already exist + auth_info = self.AuthHierarchy.get_auth_info(hrn) + table = SfaTable() + auth_record = table.find({'type': 'authority', 'hrn': hrn}) + + if not auth_record: + auth_record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=-1) + auth_record['authority'] = get_authority(auth_record['hrn']) + self.logger.info("Import: inserting authority record for %s"%hrn) + table.insert(auth_record) + print>>sys.stderr, "\r\n ========= \t\t NO AUTH RECORD \r\n" ,auth_record['authority'] + + + def create_interface_records(self): + """ + Create a record for each SFA interface + """ + # just create certs for all sfa interfaces even if they + # arent enabled + interface_hrn = self.config.SFA_INTERFACE_HRN + interfaces = ['authority+sa', 'authority+am', 'authority+sm'] + table = SfaTable() + auth_info = self.AuthHierarchy.get_auth_info(interface_hrn) + pkey = auth_info.get_pkey_object() + for interface in interfaces: + interface_record = table.find({'type': interface, 'hrn': interface_hrn}) + if not interface_record: + self.logger.info("Import: interface %s %s " % (interface_hrn, interface)) + urn = hrn_to_urn(interface_hrn, interface) + gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) + record = SfaRecord(hrn=interface_hrn, gid=gid, type=interface, pointer=-1) + record['authority'] = get_authority(interface_hrn) + print>>sys.stderr,"\r\n ==========create_interface_records", record['authority'] + table.insert(record) + + def import_person(self, parent_hrn, person): + """ + Register a user record + """ + hrn = email_to_hrn(parent_hrn, person['email']) + + print >>sys.stderr , "\r\n_____00______SfaImport : person", person + # ASN.1 will have problems with hrn's longer than 64 characters + if len(hrn) > 64: + hrn = hrn[:64] + print >>sys.stderr , "\r\n_____0______SfaImport : parent_hrn", parent_hrn + self.logger.info("Import: person %s"%hrn) + key_ids = [] + if 'key_ids' in person and person['key_ids']: + key_ids = person["key_ids"] + # get the user's private key from the SSH keys they have uploaded + # to planetlab + keys = self.shell.GetKeys(self.plc_auth, key_ids) + print >>sys.stderr , "\r\n_____1______SfaImport : self.plc_auth %s \r\n \t keys %s " %(self.plc_auth,keys) + key = keys[0]['key'] + pkey = convert_public_key(key) + print >>sys.stderr , "\r\n_____2______SfaImport : key %s pkey %s"% (key,pkey.as_pem()) + if not pkey: + pkey = Keypair(create=True) + else: + # the user has no keys + self.logger.warning("Import: person %s does not have a PL public key"%hrn) + # if a key is unavailable, then we still need to put something in the + # user's GID. So make one up. + pkey = Keypair(create=True) + print >>sys.stderr , "\r\n___ELSE________SfaImport pkey : %s \r\n \t pkey.key.bits%s "%(dir(pkey.key), pkey.as_pem()) + # create the gid + urn = hrn_to_urn(hrn, 'user') + print >>sys.stderr , "\r\n \t\t : urn ", urn + person_gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) + table = SfaTable() + person_record = SfaRecord(hrn=hrn, gid=person_gid, type="user", pointer=person['person_id']) + person_record['authority'] = get_authority(person_record['hrn']) + existing_records = table.find({'hrn': hrn, 'type': 'user', 'pointer': person['person_id']}) + if not existing_records: + table.insert(person_record) + else: + self.logger.info("Import: %s exists, updating " % hrn) + existing_record = existing_records[0] + person_record['record_id'] = existing_record['record_id'] + table.update(person_record) + + def import_slice(self, parent_hrn, slice): + slicename = slice['name'].split("_",1)[-1] + slicename = _cleanup_string(slicename) + + if not slicename: + self.logger.error("Import: failed to parse slice name %s" %slice['name']) + return + + hrn = parent_hrn + "." + slicename + self.logger.info("Import: slice %s"%hrn) + + pkey = Keypair(create=True) + urn = hrn_to_urn(hrn, 'slice') + slice_gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) + slice_record = SfaRecord(hrn=hrn, gid=slice_gid, type="slice", pointer=slice['slice_id']) + slice_record['authority'] = get_authority(slice_record['hrn']) + table = SfaTable() + existing_records = table.find({'hrn': hrn, 'type': 'slice', 'pointer': slice['slice_id']}) + if not existing_records: + table.insert(slice_record) + else: + self.logger.info("Import: %s exists, updating " % hrn) + existing_record = existing_records[0] + slice_record['record_id'] = existing_record['record_id'] + table.update(slice_record) + + def import_node(self, hrn, node): + self.logger.info("Import: node %s" % hrn) + # ASN.1 will have problems with hrn's longer than 64 characters + if len(hrn) > 64: + hrn = hrn[:64] + + table = SfaTable() + node_record = table.find({'type': 'node', 'hrn': hrn}) + pkey = Keypair(create=True) + urn = hrn_to_urn(hrn, 'node') + node_gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) + node_record = SfaRecord(hrn=hrn, gid=node_gid, type="node", pointer=node['node_id']) + node_record['authority'] = get_authority(node_record['hrn']) + existing_records = table.find({'hrn': hrn, 'type': 'node', 'pointer': node['node_id']}) + if not existing_records: + table.insert(node_record) + else: + self.logger.info("Import: %s exists, updating " % hrn) + existing_record = existing_records[0] + node_record['record_id'] = existing_record['record_id'] + table.update(node_record) + + + def import_site(self, parent_hrn, site): + shell = self.shell + plc_auth = self.plc_auth + print >>sys.stderr , " \r\n !!!!!!!!! import_site plc_shell %s \r\n \t type %s dir %s" %(shell, type(shell),dir(shell)) + sitename = site['login_base'] + sitename = _cleanup_string(sitename) + hrn = parent_hrn + "." + sitename + + # Hardcode 'internet2' into the hrn for sites hosting + # internet2 nodes. This is a special operation for some vini + # sites only + if ".vini" in parent_hrn and parent_hrn.endswith('vini'): + if sitename.startswith("i2"): + #sitename = sitename.replace("ii", "") + hrn = ".".join([parent_hrn, "internet2", sitename]) + elif sitename.startswith("nlr"): + #sitename = sitename.replace("nlr", "") + hrn = ".".join([parent_hrn, "internet2", sitename]) + + urn = hrn_to_urn(hrn, 'authority') + self.logger.info("Import: site %s"%hrn) + print >>sys.stderr , " \r\n !!!!!!!!! import_site sitename %s sitename %s \r\n \t hrn %s urn %s" %(site['login_base'],sitename, hrn,urn) + # create the authority + if not self.AuthHierarchy.auth_exists(urn): + self.AuthHierarchy.create_auth(urn) + + auth_info = self.AuthHierarchy.get_auth_info(urn) + + table = SfaTable() + auth_record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=site['site_id']) + auth_record['authority'] = get_authority(auth_record['hrn']) + existing_records = table.find({'hrn': hrn, 'type': 'authority', 'pointer': site['site_id']}) + if not existing_records: + table.insert(auth_record) + else: + self.logger.info("Import: %s exists, updating " % hrn) + existing_record = existing_records[0] + auth_record['record_id'] = existing_record['record_id'] + table.update(auth_record) + + return hrn + + + def delete_record(self, hrn, type): + # delete the record + table = SfaTable() + record_list = table.find({'type': type, 'hrn': hrn}) + for record in record_list: + self.logger.info("Import: removing record %s %s" % (type, hrn)) + table.remove(record)