python setup.py install --root=$(DESTDIR)
chmod 444 $(DESTDIR)/etc/sfa/default_config.xml
rm -rf $(DESTDIR)/usr/lib*/python*/site-packages/*egg-info
- rm -rf $(DESTDIR)/usr/lib*/python*/site-packages/sfa/storage/sfa.sql
rm -rf $(DESTDIR)/usr/lib*/python*/site-packages/sfa/storage/migrations
(cd $(DESTDIR)/usr/bin ; ln -s sfi.py sfi; ln -s sfascan.py sfascan)
@exit 1
endif
-sync: synccheck
+
+synclib: synccheck
+$(RSYNC) --relative ./sfa/ $(SSHURL)/usr/lib\*/python2.\*/site-packages/
- +$(RSYNC) ./tests/ $(SSHURL)/root/tests-sfa
+syncbin: synccheck
+$(RSYNC) $(BINS) $(SSHURL)/usr/bin/
+syncinit: synccheck
+$(RSYNC) ./init.d/sfa $(SSHURL)/etc/init.d/
+syncconfig:
+$(RSYNC) ./config/default_config.xml $(SSHURL)/etc/sfa/
- +$(RSYNC) ./sfa/storage/{sfa.sql,migrations} $(SSHURL)/usr/share/sfa/
+synctest: synccheck
+ +$(RSYNC) ./tests/ $(SSHURL)/root/tests-sfa
+syncrestart: synccheck
$(SSHCOMMAND) exec service sfa restart
+syncmig:
+ +$(RSYNC) ./sfa/storage/migrations $(SSHURL)/usr/share/sfa/
+
+
+# full-fledged
+sync: synclib syncbin syncinit syncconfig syncrestart
# 99% of the time this is enough
-fastsync: synccheck
- +$(RSYNC) --relative ./sfa/ $(SSHURL)/usr/lib\*/python2.\*/site-packages/
- $(SSHCOMMAND) exec service sfa restart
+syncfast: synclib syncrestart
-clientsync: synccheck
- +$(RSYNC) $(BINS) $(SSHURL)/usr/bin/
+.PHONY: synccheck synclib syncbin syncconfig synctest syncrestart sync syncfast
-ricasync: synccheck
+syncrica: synccheck
+$(RSYNC) --relative ./sfa/fd ./sfa/generic/fd.py ./sfa/rspecs/versions/federica.py $(SSHURL)/usr/lib\*/python2.\*/site-packages/
$(SSHCOMMAND) exec service sfa restart
-.PHONY: synccheck sync fastsync clientsync ricasync
+.PHONY: syncrica
##########
CLIENTLIBFILES= \
DATE=$(date +"%Y-%m-%d-%H-%M-%S")
# Dumps the database - optional argument to specify filename suffix
-function dump_sfa_db()
-{
+function dump_sfa_db() {
if [ -n "$1" ] ; then suffix="-$1" ; else suffix="" ; fi
dumpfile=/var/lib/pgsql/backups/$(date +"${SFA_DB_NAME}.${DATE}${suffix}.sql")
pg_dump -U $SFA_DB_USER $SFA_DB_NAME > $dumpfile
check
}
-# Updates the database by applying all migration scripts in
-# /usr/share/sfa/migrations/N-up-*, where N is greater than the
-# current subversion. At least one of the migration scripts with the
-# same N must update sfa_db_version.subversion.
-function migrate_db()
-{
- subversion=$(psql -U $SFA_DB_USER --quiet --tuples-only --no-align -c \
- "SELECT subversion FROM sfa_db_version LIMIT 1" \
- $SFA_DB_NAME 2>/dev/null || echo 0)
- shopt -s nullglob
- for file in /usr/share/sfa/migrations/[0-9]*-up-* ; do
- script=$(basename $file)
- index=${script%-up*}
- extension=${script##*.}
- if [ $index -gt $subversion ] ; then
- if [ "$extension" = "sql" ] ; then
- dialog " - $script (dbdumped)"
- dump_sfa_db "before-$script"
- psql -U $SFA_DB_USER -f $file $SFA_DB_NAME
- elif [ -x $file ] ; then
- dialog " - $script (dbdumped)"
- dump_sfa_db "before-$script"
- $file
- else
- dialog "\nWarning: migration $file not executable"
- fi
- check
- fi
- done
-}
+# Updates the database by applying migration scripts
+# using sqlalchemy-migrate for this
+# migrate repo is /usr/share/sfa/migrations/
+# and version scripts are thus in /usr/share/sfa/migrations/versions/
+function db_migrate() {
+ # only if enabled
+ [ "$SFA_DB_ENABLED" == 1 ] || return
+ alchemy_url="postgresql+psycopg2://${SFA_DB_USER}:${SFA_DB_PASSWORD}@:${SFA_DB_PORT}/${SFA_DB_NAME}"
+ migrate_repo="/usr/share/sfa/migrations"
+ # check if under version control, and initialize it otherwise
+ if ! sqlalchemy-migrate db_version --url=$alchemy_url $migrate_repo >& /dev/null ; then
+ # at this point it would make sense to
+ # check that the 'users' table is in the db
+ # if not, backup/dump, and run a nuke/import
+ # and then only init the db version to 0
+ echo -n $"version init in db ${SFA_DB_NAME}"
+ sqlalchemy-migrate version_control --url=$alchemy_url $migrate_repo
+ fi
+ version_before=$(sqlalchemy-migrate db_version --url=$alchemy_url $migrate_repo)
+ check
+ sqlalchemy-migrate upgrade --url=$alchemy_url $migrate_repo 2> /dev/null
+ check
+ version_after=$(sqlalchemy-migrate db_version --url=$alchemy_url $migrate_repo)
+ if [ "$version_before" != "$version_after" -o "$ERRORS" != 0 ] ; then
+ MESSAGE=$"DB version: $version_before -> $version_after"
+ echo -n "$MESSAGE"
+ [ "$ERRORS" == 0 ] && success "$MESSAGE" || failure "$MESSAGE" ; echo
+ fi
+}
# Regenerate configuration files - almost verbatim from plc.init
function reload () {
echo "host $SFA_DB_NAME $SFA_DB_USER 127.0.0.1/32 password"
[ -n "$registry_ip" ] && echo "host $SFA_DB_NAME $SFA_DB_USER ${registry_ip}/32 password"
) >>$pghba_conf
+
+ if [ "$SFA_GENERIC_FLAVOUR" == "openstack" ] ; then
+ [ -n "$registry_ip" ] && echo "host nova nova ${registry_ip}/32 password" >> $pghba_conf
+ fi
# Fix ownership (sed -i changes it)
chown postgres:postgres $postgresql_conf $pghba_conf
if ! psql -U $SFA_DB_USER -c "" $SFA_DB_NAME >/dev/null 2>&1 ; then
createdb -U postgres --template=template0 --encoding=UNICODE --owner=$SFA_DB_USER $SFA_DB_NAME
check
- # install db schema
- psql -U $SFA_DB_USER -f /usr/share/sfa/sfa.sql $SFA_DB_NAME
- check
fi
check
reload
db_start
-
- migrate_db
+ db_migrate
# install peer certs
action $"SFA installing peer certs" daemon /usr/bin/sfa-start.py -t -d $OPTIONS
packages = [
'sfa',
+ 'sfa/openstack',
'sfa/trust',
'sfa/storage',
'sfa/util',
('/etc/sfatables/matches/', glob('sfatables/matches/*.xml')),
('/etc/sfatables/targets/', glob('sfatables/targets/*.xml')),
('/etc/init.d/', [ "init.d/%s"%x for x in initscripts ]),
- ('/usr/share/sfa/', [ 'sfa/storage/sfa.sql' ] ),
('/usr/share/sfa/migrations', [ 'sfa/storage/migrations/*' ] ),
+ ('/usr/share/sfa/migrations/versions', [ 'sfa/storage/migrations/versions/*' ] ),
('/usr/share/sfa/examples/', glob('sfa/examples/*' ) + [ 'cron.d/sfa.cron' ] ),
]
%define name sfa
%define version 2.0
-%define taglevel 8
+%define taglevel 9
%define release %{taglevel}%{?pldistro:.%{pldistro}}%{?date:.%{date}}
%global python_sitearch %( python -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)" )
Summary: the SFA python libraries
Group: Applications/System
BuildRequires: make
+
+Requires: myplc-config
Requires: python >= 2.5
+Requires: pyOpenSSL >= 0.7
Requires: m2crypto
Requires: xmlsec1-openssl-devel
Requires: libxslt-python
Requires: postgresql >= 8.2, postgresql-server >= 8.2
Requires: postgresql-python
Requires: python-psycopg2
-Requires: pyOpenSSL >= 0.7
-Requires: myplc-config
+# f8=0.4 - f12=0.5 f14=0.6 f16=0.7
+Requires: python-sqlalchemy
+Requires: python-migrate
+# the eucalyptus aggregate uses this module
Requires: python-xmlbuilder
# python 2.5 has uuid module added, for python 2.4 we still need it.
%config /etc/sfa/default_config.xml
%config (noreplace) /etc/sfa/aggregates.xml
%config (noreplace) /etc/sfa/registries.xml
-/usr/share/sfa/sfa.sql
/usr/share/sfa/examples
/var/www/html/wsdl/*.wsdl
[ "$1" -ge "1" ] && service sfa-cm restart || :
%changelog
+* Sat Jan 7 2012 Tony Mack <tmack@cs.princeton.edu> - sfa-2.0-9
+- bugfix: 'geni_api' should be in the top level struct, not the code struct
+- bugfix: Display the correct host and port in 'geni_api_versions' field of the GetVersion
+ output returned by the Aggregate Manager.
+- bugfix: sfa.util.sfatime now handles numeric string inputs correctly.
+- bugfix: sfa.util.sfatime.datetime_to_epoch() returns integers instead of doubles.
+- bugfix: Fixed bug that prevented the rspec parser from identifying an rspec's schema when
+ there is extra whitespace in the schemaLocation field.
+- bugfix: Fixed bug that caused PlanetLab initscripts from showing up in the PGv2 and GENIv3
+ advertisement rspecs.
+- bugfix: <login> RSpec element should contain the 'username' attribute.
+- bugfix: Use sfa.util.plxrn.PlXrn to parse the login_base (authority) out of a urn.
+
* Wed Jan 4 2012 Tony Mack <tmack@cs.princeton.edu> - sfa-2.0-8
- bugfix: Fixed a bug in the sfa-import-plc.py script that caused the script to
exit when it encountered a user with an invalid public key.
from optparse import OptionParser
+from sfa.client.return_value import ReturnValue
from sfa.client.sfi import Sfi
from sfa.util.sfalogging import logger, DEBUG
from sfa.client.sfaserverproxy import SfaServerProxy
### assuming everything is sequential, as simple as it gets
### { url -> (timestamp,version)}
class VersionCache:
+ # default expiration period is 1h
def __init__ (self, filename=None, expires=60*60):
# default is to store cache in the same dir as argv[0]
if filename is None:
return None
###
+# non-existing hostnames happen...
+# for better perfs we cache the result of gethostbyname too
class Interface:
- def __init__ (self,url,verbose=False):
+ def __init__ (self,url,mentioned_in=None,verbose=False):
self._url=url
self.verbose=verbose
+ cache=VersionCache()
+ key="interface:%s"%url
try:
(self._url,self.hostname,self.port)=url_hostname_port(url)
- self.ip=socket.gethostbyname(self.hostname)
- self.probed=False
+ # look for ip in the cache
+ tuple=cache.get(key)
+ if tuple:
+ (self.hostname, self.ip, self.port) = tuple
+ else:
+ self.ip=socket.gethostbyname(self.hostname)
except:
+ msg="can't resolve hostname %s\n\tfound in url %s"%(self.hostname,self._url)
+ if mentioned_in:
+ msg += "\n\t(mentioned at %s)"%mentioned_in
+ logger.warning (msg)
self.hostname="unknown"
self.ip='0.0.0.0'
self.port="???"
+
+ cache.set(key, (self.hostname, self.ip, self.port,) )
+ cache.save()
+ self.probed=False
+
+ # mark unknown interfaces as probed to avoid unnecessary attempts
+ if self.hostname=='unknown':
# don't really try it
self.probed=True
self._version={}
+
def url(self):
return self._url
# setting timeout here seems to get the call to fail - even though the response time is fast
#server=SfaServerProxy(url, key_file, cert_file, verbose=self.verbose, timeout=options.timeout)
server=SfaServerProxy(url, key_file, cert_file, verbose=self.verbose)
- self._version=server.GetVersion()
+ self._version=ReturnValue.get_value(server.GetVersion())
except:
logger.log_exc("failed to get version")
self._version={}
try: shape=Interface.shapes[version['interface']]
except: shape=Interface.shapes['default']
layout['shape']=shape
- ### fill color to outline wrongly configured bodies
- if 'geni_api' not in version and 'sfa' not in version:
+ ### fill color to outline wrongly configured or unreachable bodies
+ # as of sfa-2.0 registry doesn't have 'sfa' not 'geni_api', but have peers
+ # slicemgr and aggregate have 'geni_api' and 'sfa'
+ if 'geni_api' not in version and 'peers' not in version:
layout['style']='filled'
layout['fillcolor']='gray'
return layout
logger.debug(k)
for (k1,v1) in v.iteritems():
logger.debug("\r\t\t%s:%s"%(k1,v1))
- # 'geni_api' is expected if the call succeeded at all
- # 'peers' is needed as well as AMs typically don't have peers
- if 'geni_api' in version and 'peers' in version:
- # proceed with neighbours
+ # proceed with neighbours
+ if 'peers' in version:
for (next_name,next_url) in version['peers'].iteritems():
- next_interface=Interface(next_url)
+ next_interface=Interface(next_url,mentioned_in=interface.url())
# locate or create node in graph
try:
# if found, we're good with this one
if not options.outfiles:
options.outfiles=SfaScan.default_outfiles
scanner=Scanner(left_to_right=options.left_to_right, verbose=bool_verbose)
- entries = [ Interface(entry) for entry in args ]
+ entries = [ Interface(entry,mentioned_in="command line") for entry in args ]
try:
g=scanner.graph(entries)
logger.info("creating layout")
from sfa.util.version import version_core
from sfa.util.cache import Cache
-from sfa.storage.record import SfaRecord, UserRecord, SliceRecord, NodeRecord, AuthorityRecord
+from sfa.storage.persistentobjs import RegRecord, RegAuthority, RegUser, RegSlice, RegNode
+from sfa.storage.persistentobjs import make_record
from sfa.rspecs.rspec import RSpec
from sfa.rspecs.rspec_converter import RSpecConverter
f.close()
return
-def save_records_to_file(filename, recordList, format="xml"):
+def save_records_to_file(filename, record_dicts, format="xml"):
if format == "xml":
index = 0
- for record in recordList:
+ for record_dict in record_dicts:
if index > 0:
- save_record_to_file(filename + "." + str(index), record)
+ save_record_to_file(filename + "." + str(index), record_dict)
else:
- save_record_to_file(filename, record)
+ save_record_to_file(filename, record_dict)
index = index + 1
elif format == "xmllist":
f = open(filename, "w")
f.write("<recordlist>\n")
- for record in recordList:
- record = SfaRecord(dict=record)
- f.write('<record hrn="' + record.get_name() + '" type="' + record.get_type() + '" />\n')
+ for record_dict in record_dicts:
+ record_obj=make_record (dict=record_dict)
+ f.write('<record hrn="' + record_obj.hrn + '" type="' + record_obj.type + '" />\n')
f.write("</recordlist>\n")
f.close()
elif format == "hrnlist":
f = open(filename, "w")
- for record in recordList:
- record = SfaRecord(dict=record)
- f.write(record.get_name() + "\n")
+ for record_dict in record_dicts:
+ record_obj=make_record (dict=record_dict)
+ f.write(record_obj.hrn + "\n")
f.close()
else:
# this should never happen
print "unknown output format", format
-def save_record_to_file(filename, record):
- if record['type'] in ['user']:
- record = UserRecord(dict=record)
- elif record['type'] in ['slice']:
- record = SliceRecord(dict=record)
- elif record['type'] in ['node']:
- record = NodeRecord(dict=record)
- elif record['type'] in ['authority', 'ma', 'sa']:
- record = AuthorityRecord(dict=record)
- else:
- record = SfaRecord(dict=record)
+def save_record_to_file(filename, record_dict):
+ rec_record = make_record (dict=record_dict)
str = record.save_to_string()
f=codecs.open(filename, encoding='utf-8',mode="w")
f.write(str)
# load methods
def load_record_from_file(filename):
f=codecs.open(filename, encoding="utf-8", mode="r")
- str = f.read()
+ xml_string = f.read()
f.close()
- record = SfaRecord(string=str)
- return record
+ return make_record (xml=xml_string)
import uuid
help="root registry", metavar="URL", default=None)
parser.add_option("-s", "--sliceapi", dest="sm", default=None, metavar="URL",
help="slice API - in general a SM URL, but can be used to talk to an aggregate")
+ parser.add_option("-R", "--raw", dest="raw", action="store_true", default=False,
+ help="Display raw, unparsed server response")
parser.add_option("-d", "--dir", dest="sfi_dir",
help="config & working directory - default is %default",
metavar="PATH", default=Sfi.default_sfi_dir())
self.print_help()
sys.exit(1)
hrn = args[0]
- records = self.registry().Resolve(hrn, self.my_credential_string)
- records = filter_records(options.type, records)
- if not records:
+ record_dicts = self.registry().Resolve(hrn, self.my_credential_string)
+ record_dicts = filter_records(options.type, record_dicts)
+ if not record_dicts:
self.logger.error("No record of type %s"% options.type)
+ records = [ make_record (dict=record_dict) for record_dict in record_dicts ]
for record in records:
- if record['type'] in ['user']:
- record = UserRecord(dict=record)
- elif record['type'] in ['slice']:
- record = SliceRecord(dict=record)
- elif record['type'] in ['node']:
- record = NodeRecord(dict=record)
- elif record['type'].startswith('authority'):
- record = AuthorityRecord(dict=record)
- else:
- record = SfaRecord(dict=record)
- if (options.format == "text"):
- record.dump()
- else:
- print record.save_to_string()
+ if (options.format == "text"): record.dump()
+ else: print record.save_as_xml()
if options.file:
- save_records_to_file(options.file, records, options.fileformat)
+ save_records_to_file(options.file, record_dicts, options.fileformat)
return
def add(self, options, args):
sys.exit(1)
record_filepath = args[0]
rec_file = self.get_record_file(record_filepath)
- record = load_record_from_file(rec_file).as_dict()
+ record = load_record_from_file(rec_file).todict()
return self.registry().Register(record, auth_cred)
def update(self, options, args):
sys.exit(1)
rec_file = self.get_record_file(args[0])
record = load_record_from_file(rec_file)
- if record['type'] == "user":
- if record.get_name() == self.user:
+ if record.type == "user":
+ if record.hrn == self.user:
cred = self.my_credential_string
else:
cred = self.my_authority_credential_string()
- elif record['type'] in ["slice"]:
+ elif record.type in ["slice"]:
try:
- cred = self.slice_credential_string(record.get_name())
+ cred = self.slice_credential_string(record.hrn)
except ServerException, e:
# XXX smbaker -- once we have better error return codes, update this
# to do something better than a string compare
cred = self.my_authority_credential_string()
else:
raise
- elif record.get_type() in ["authority"]:
+ elif record.type in ["authority"]:
cred = self.my_authority_credential_string()
- elif record.get_type() == 'node':
+ elif record.type == 'node':
cred = self.my_authority_credential_string()
else:
- raise "unknown record type" + record.get_type()
- record = record.as_dict()
- return self.registry().Update(record, cred)
+ raise "unknown record type" + record.type
+ record_dict = record.todict()
+ return self.registry().Update(record_dict, cred)
def remove(self, options, args):
"remove registry record by name (Remove)"
api_options['call_id']=unique_call_id()
result = server.ListSlices(creds, *self.ois(server,api_options))
value = ReturnValue.get_value(result)
- display_list(value)
+ if self.options.raw:
+ print result
+ else:
+ display_list(value)
return
# show rspec for named slice
result = server.ListResources (creds, api_options)
value = ReturnValue.get_value(result)
if options.file is None:
- display_rspec(value, options.format)
+ if self.options.raw:
+ print result
+ else:
+ display_rspec(value, options.format)
else:
save_rspec_to_file(value, options.file)
return
result = server.CreateSliver(slice_urn, creds, rspec, users, *self.ois(server, api_options))
value = ReturnValue.get_value(result)
if options.file is None:
- print value
+ if self.options.raw:
+ print result
+ else:
+ print value
else:
save_rspec_to_file (value, options.file)
return value
api_options = {}
api_options ['call_id'] = unique_call_id()
result = server.DeleteSliver(slice_urn, creds, *self.ois(server, api_options ) )
- # xxx no ReturnValue ??
- return result
+ value = ReturnValue.get_value(result)
+ if self.options.raw:
+ print result
+ else:
+ print value
+ return value
def status(self, options, args):
"""
# options and call_id when supported
api_options = {}
- api_options['call_id']=unique_call_id()
+ api_options['call_id']=unique_call_id()
result = server.SliverStatus(slice_urn, creds, *self.ois(server,api_options))
value = ReturnValue.get_value(result)
- print value
+ if self.options.raw:
+ print result
+ else:
+ print value
if options.file:
save_variable_to_file(value, options.file, options.fileformat)
delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
creds.append(delegated_cred)
# xxx Thierry - does this not need an api_options as well ?
- return server.Start(slice_urn, creds)
+ result = server.Start(slice_urn, creds)
+ value = ReturnValue.get_value(result)
+ if self.options.raw:
+ print result
+ else:
+ print value
+ return value
def stop(self, options, args):
"""
if options.delegate:
delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
creds.append(delegated_cred)
- return server.Stop(slice_urn, creds)
+ result = server.Stop(slice_urn, creds)
+ value = ReturnValue.get_value(result)
+ if self.options.raw:
+ print result
+ else:
+ print value
+ return value
# reset named slice
def reset(self, options, args):
if options.delegate:
delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
creds.append(delegated_cred)
- return server.reset_slice(creds, slice_urn)
+ result = server.reset_slice(creds, slice_urn)
+ value = ReturnValue.get_value(result)
+ if self.options.raw:
+ print result
+ else:
+ print value
+ return value
def renew(self, options, args):
"""
api_options['call_id']=unique_call_id()
result = server.RenewSliver(slice_urn, creds, time, *self.ois(server,api_options))
value = ReturnValue.get_value(result)
+ if self.options.raw:
+ print result
+ else:
+ print value
return value
if options.delegate:
delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority))
creds.append(delegated_cred)
- return server.Shutdown(slice_urn, creds)
+ result = server.Shutdown(slice_urn, creds)
+ value = ReturnValue.get_value(result)
+ if self.options.raw:
+ print result
+ else:
+ print value
+ return value
def get_ticket(self, options, args):
--- /dev/null
+from sfa.generic import Generic
+
+import sfa.server.sfaapi
+import sfa.openstack.openstack_driver
+import sfa.managers.registry_manager_openstack
+import sfa.managers.slice_manager
+import sfa.managers.aggregate_manager_openstack
+
+class openstack (Generic):
+
+ # use the standard api class
+ def api_class (self):
+ return sfa.server.sfaapi.SfaApi
+
+ # the manager classes for the server-side services
+ def registry_manager_class (self) :
+ return sfa.managers.registry_manager_openstack.RegistryManager
+ def slicemgr_manager_class (self) :
+ return sfa.managers.slice_manager.SliceManager
+ def aggregate_manager_class (self) :
+ return sfa.managers.aggregate_manager_openstack.AggregateManager
+
+ # driver class for server-side services, talk to the whole testbed
+ def driver_class (self):
+ return sfa.openstack.openstack_driver.OpenstackDriver
+
+ # for the component mode, to be run on board planetlab nodes
+ # manager class
+ def component_manager_class (self):
+ return sfa.managers.component_manager_pl
+ # driver_class
+ def component_driver_class (self):
+ return sfa.plc.plcomponentdriver.PlComponentDriver
+
+
from sfa.generic import Generic
-import sfa.server.sfaapi
-import sfa.plc.pldriver
-import sfa.managers.registry_manager
-import sfa.managers.slice_manager
-import sfa.managers.aggregate_manager
class pl (Generic):
# use the standard api class
def api_class (self):
+ import sfa.server.sfaapi
return sfa.server.sfaapi.SfaApi
# the manager classes for the server-side services
def registry_manager_class (self) :
+ import sfa.managers.registry_manager
return sfa.managers.registry_manager.RegistryManager
def slicemgr_manager_class (self) :
+ import sfa.managers.slice_manager
return sfa.managers.slice_manager.SliceManager
def aggregate_manager_class (self) :
+ import sfa.managers.aggregate_manager
return sfa.managers.aggregate_manager.AggregateManager
# driver class for server-side services, talk to the whole testbed
def driver_class (self):
+ import sfa.plc.pldriver
return sfa.plc.pldriver.PlDriver
# for the component mode, to be run on board planetlab nodes
from sfa.util.config import Config
from sfa.util.xrn import Xrn, get_leaf, get_authority, hrn_to_urn
from sfa.util.plxrn import hostname_to_hrn, slicename_to_hrn, email_to_hrn, hrn_to_pl_slicename
-from sfa.storage.table import SfaTable
-from sfa.storage.record import SfaRecord
+from sfa.util.sfalogging import logger
+
+from sfa.trust.gid import create_uuid
from sfa.trust.certificate import convert_public_key, Keypair
-from sfa.trust.gid import create_uuid
+
+from sfa.openstack.openstack_shell import OpenstackShell
+
+from sfa.storage.alchemy import dbsession
+from sfa.storage.persistentobjs import RegRecord, RegAuthority, RegUser, RegSlice, RegNode
+
from sfa.importer.sfaImport import sfaImport, _cleanup_string
-from sfa.util.sfalogging import logger
-try:
- from nova.auth.manager import AuthManager, db, context
-except ImportError:
- AuthManager = None
-
def process_options():
sys.exit(0)
root_auth = config.SFA_REGISTRY_ROOT_AUTH
interface_hrn = config.SFA_INTERFACE_HRN
- if AuthManager:
- auth_manager = AuthManager()
- else:
- logger.info("Unable to import nova.auth.manager. Doesn't look like openstack-copute is installed. Exiting...")
- sys.exit(0)
+ shell = OpenstackShell(config)
sfaImporter.create_top_level_records()
# create dict of all existing sfa records
existing_records = {}
existing_hrns = []
key_ids = []
- table = SfaTable()
- results = table.find()
- for result in results:
- existing_records[(result['hrn'], result['type'])] = result
- existing_hrns.append(result['hrn'])
+ for record in dbsession.query(RegRecord):
+ existing_records[ (record.hrn, record.type,) ] = record
+ existing_hrns.append(record.hrn)
# Get all users
- persons = auth_manager.get_users()
+ persons = shell.user_get_all()
persons_dict = {}
keys_filename = config.config_path + os.sep + 'person_keys.py'
- old_person_keys = load_keys(keys_filename)
+ old_person_keys = load_keys(keys_filename)
person_keys = {}
for person in persons:
hrn = config.SFA_INTERFACE_HRN + "." + person.id
+ persons_dict[hrn] = person
old_keys = old_person_keys.get(person.id, [])
- keys = db.key_pair_get_all_by_user(context.get_admin_context(), person.id)
- person_keys[person.id] = [key.public_key for key in keys]
+ keys = [k.public_key for k in shell.key_pair_get_all_by_user(person.id)]
+ person_keys[person.id] = keys
update_record = False
if old_keys != keys:
update_record = True
if hrn not in existing_hrns or \
(hrn, 'user') not in existing_records or update_record:
urn = hrn_to_urn(hrn, 'user')
+
if keys:
try:
- pkey = convert_public_key(key)
+ pkey = convert_public_key(keys[0])
except:
- logger.warn('unable to convert public key for %s' % hrn)
+ logger.log_exc('unable to convert public key for %s' % hrn)
pkey = Keypair(create=True)
else:
logger.warn("Import: person %s does not have a PL public key"%hrn)
pkey = Keypair(create=True)
- person_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
- person_record = SfaRecord(hrn=hrn, gid=person_gid, type="user", \
- authority=get_authority(hrn))
- persons_dict[person_record['hrn']] = person_record
- person_record.sync()
+ person_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
+ person_record = RegUser ()
+ person_record.type='user'
+ person_record.hrn=hrn
+ person_record.gid=person_gid
+ person_record.authority=get_authority(hrn)
+ dbsession.add(person_record)
+ dbsession.commit()
+ logger.info("Import: imported person %s" % person_record)
# Get all projects
- projects = db.project_get_all(context.get_admin_context())
+ projects = shell.project_get_all()
projects_dict = {}
for project in projects:
hrn = config.SFA_INTERFACE_HRN + '.' + project.id
+ projects_dict[hrn] = project
if hrn not in existing_hrns or \
(hrn, 'slice') not in existing_records:
pkey = Keypair(create=True)
urn = hrn_to_urn(hrn, 'slice')
project_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
- project_record = SfaRecord(hrn=hrn, gid=project_gid, type="slice",
- authority=get_authority(hrn))
- projects_dict[project_record['hrn']] = project_record
- project_record.sync(verbose=True)
+ project_record = RegSlice ()
+ project_record.type='slice'
+ project_record.hrn=hrn
+ project_record.gid=project_gid
+ project_record.authority=get_authority(hrn)
+ dbsession.add(project_record)
+ dbsession.commit()
+ logger.info("Import: imported slice: %s" % project_record)
# remove stale records
system_records = [interface_hrn, root_auth, interface_hrn + '.slicemanager']
continue
record = existing_records[(record_hrn, type)]
- if record['peer_authority']:
+ if record.peer_authority:
continue
if type == 'user':
else:
continue
- record_object = existing_records[(record_hrn, type)]
- record = SfaRecord(dict=record_object)
- record.delete()
+ record_object = existing_records[ (record_hrn, type) ]
+ logger.info("Import: removing %s " % record)
+ dbsession.delete(record_object)
+ dbsession.commit()
# save pub keys
logger.info('Import: saving current pub keys')
from sfa.util.config import Config
from sfa.util.xrn import Xrn, get_leaf, get_authority, hrn_to_urn
from sfa.util.plxrn import hostname_to_hrn, slicename_to_hrn, email_to_hrn, hrn_to_pl_slicename
-from sfa.storage.table import SfaTable
-from sfa.storage.record import SfaRecord
+
from sfa.trust.gid import create_uuid
from sfa.trust.certificate import convert_public_key, Keypair
-from sfa.importer.sfaImport import sfaImport, _cleanup_string
+
from sfa.plc.plshell import PlShell
+from sfa.storage.alchemy import dbsession
+from sfa.storage.persistentobjs import RegRecord, RegAuthority, RegUser, RegSlice, RegNode
+
+from sfa.importer.sfaImport import sfaImport, _cleanup_string
+
def process_options():
(options, args) = getopt.getopt(sys.argv[1:], '', [])
hrn = ".".join([interface_hrn, "internet2", site['login_base']])
return hrn
+# maybe could use a call to persistentobjs.init_tables somewhere
+# however now import s expected to be done after service creation..
def main():
process_options()
config = Config()
+ sfaImporter = sfaImport()
+ logger=sfaImporter.logger
+ logger.setLevelFromOptVerbose(config.SFA_API_LOGLEVEL)
if not config.SFA_REGISTRY_ENABLED:
sys.exit(0)
root_auth = config.SFA_REGISTRY_ROOT_AUTH
interface_hrn = config.SFA_INTERFACE_HRN
- keys_filename = config.config_path + os.sep + 'person_keys.py'
- sfaImporter = sfaImport()
- sfaImporter.create_top_level_records()
- logger=sfaImporter.logger
- logger.setLevelFromOptVerbose(config.SFA_API_LOGLEVEL)
shell = PlShell (config)
+ sfaImporter.create_top_level_records()
# create dict of all existing sfa records
existing_records = {}
existing_hrns = []
key_ids = []
- person_keys = {}
- table = SfaTable()
- results = table.find()
- for result in results:
- existing_records[(result['hrn'], result['type'])] = result
- existing_hrns.append(result['hrn'])
+ for record in dbsession.query(RegRecord):
+ existing_records[ (record.hrn, record.type,) ] = record
+ existing_hrns.append(record.hrn)
# Get all plc sites
sites = shell.GetSites({'peer_id': None})
keys_dict[key['key_id']] = key['key']
# create a dict of person keys keyed on key_id
+ keys_filename = config.config_path + os.sep + 'person_keys.py'
old_person_keys = load_keys(keys_filename)
+ person_keys = {}
for person in persons:
pubkeys = []
for key_id in person['key_ids']:
i2site = {'name': 'Internet2', 'abbreviated_name': 'I2',
'login_base': 'internet2', 'site_id': -1}
site_hrn = _get_site_hrn(interface_hrn, i2site)
- logger.info("Importing site: %s" % site_hrn)
# import if hrn is not in list of existing hrns or if the hrn exists
# but its not a site record
if site_hrn not in existing_hrns or \
(site_hrn, 'authority') not in existing_records:
- logger.info("Import: site %s " % site_hrn)
urn = hrn_to_urn(site_hrn, 'authority')
if not sfaImporter.AuthHierarchy.auth_exists(urn):
sfaImporter.AuthHierarchy.create_auth(urn)
auth_info = sfaImporter.AuthHierarchy.get_auth_info(urn)
- auth_record = SfaRecord(hrn=site_hrn, gid=auth_info.get_gid_object(), \
- type="authority", pointer=site['site_id'],
- authority=get_authority(site_hrn))
- auth_record.sync(verbose=True)
+ auth_record = RegAuthority()
+ auth_record.type='authority'
+ auth_record.hrn=site_hrn
+ auth_record.gid=auth_info.get_gid_object()
+ auth_record.pointer=site['site_id']
+ auth_record.authority=get_authority(site_hrn)
+ dbsession.add(auth_record)
+ dbsession.commit()
+ logger.info("Import: Imported authority (vini site) %s"%auth_record)
# start importing
for site in sites:
site_hrn = _get_site_hrn(interface_hrn, site)
- logger.info("Importing site: %s" % site_hrn)
# import if hrn is not in list of existing hrns or if the hrn exists
# but its not a site record
if site_hrn not in existing_hrns or \
(site_hrn, 'authority') not in existing_records:
try:
- logger.info("Import: site %s " % site_hrn)
urn = hrn_to_urn(site_hrn, 'authority')
if not sfaImporter.AuthHierarchy.auth_exists(urn):
sfaImporter.AuthHierarchy.create_auth(urn)
auth_info = sfaImporter.AuthHierarchy.get_auth_info(urn)
- auth_record = SfaRecord(hrn=site_hrn, gid=auth_info.get_gid_object(), \
- type="authority", pointer=site['site_id'],
- authority=get_authority(site_hrn))
- logger.info("Import: importing site: %s" % auth_record.summary_string())
- auth_record.sync()
+ auth_record = RegAuthority()
+ auth_record.type='authority'
+ auth_record.hrn=site_hrn
+ auth_record.gid=auth_info.get_gid_object()
+ auth_record.pointer=site['site_id']
+ auth_record.authority=get_authority(site_hrn)
+ dbsession.add(auth_record)
+ dbsession.commit()
+ logger.info("Import: imported authority (site) : %s" % auth_record)
except:
# if the site import fails then there is no point in trying to import the
# site's child records (node, slices, persons), so skip them.
pkey = Keypair(create=True)
urn = hrn_to_urn(hrn, 'node')
node_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
- node_record = SfaRecord(hrn=hrn, gid=node_gid, type="node", pointer=node['node_id'], authority=get_authority(hrn))
- logger.info("Import: importing node: %s" % node_record.summary_string())
- node_record.sync()
+ node_record = RegNode ()
+ node_record.type='node'
+ node_record.hrn=hrn
+ node_record.gid=node_gid
+ node_record.pointer =node['node_id']
+ node_record.authority=get_authority(hrn)
+ dbsession.add(node_record)
+ dbsession.commit()
+ logger.info("Import: imported node: %s" % node_record)
except:
logger.log_exc("Import: failed to import node")
pkey = Keypair(create=True)
urn = hrn_to_urn(hrn, 'slice')
slice_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
- slice_record = SfaRecord(hrn=hrn, gid=slice_gid, type="slice", pointer=slice['slice_id'],
- authority=get_authority(hrn))
- logger.info("Import: importing slice: %s" % slice_record.summary_string())
- slice_record.sync()
+ slice_record = RegSlice ()
+ slice_record.type='slice'
+ slice_record.hrn=hrn
+ slice_record.gid=slice_gid
+ slice_record.pointer=slice['slice_id']
+ slice_record.authority=get_authority(hrn)
+ dbsession.add(slice_record)
+ dbsession.commit()
+ logger.info("Import: imported slice: %s" % slice_record)
except:
logger.log_exc("Import: failed to import slice")
if len(hrn) > 64:
hrn = hrn[:64]
- # if user's primary key has chnaged then we need to update the
- # users gid by forcing a update here
+ # if user's primary key has changed then we need to update the
+ # users gid by forcing an update here
old_keys = []
new_keys = []
if person_id in old_person_keys:
pkey = Keypair(create=True)
urn = hrn_to_urn(hrn, 'user')
person_gid = sfaImporter.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
- person_record = SfaRecord(hrn=hrn, gid=person_gid, type="user", \
- pointer=person['person_id'], authority=get_authority(hrn))
- logger.info("Import: importing person: %s" % person_record.summary_string())
- person_record.sync()
+ person_record = RegUser ()
+ person_record.type='user'
+ person_record.hrn=hrn
+ person_record.gid=person_gid
+ person_record.pointer=person['person_id']
+ person_record.authority=get_authority(hrn)
+ dbsession.add (person_record)
+ dbsession.commit()
+ logger.info("Import: imported person: %s" % person_record)
except:
logger.log_exc("Import: failed to import person.")
continue
record = existing_records[(record_hrn, type)]
- if record['peer_authority']:
+ if record.peer_authority:
continue
# dont delete vini's internet2 placeholdder record
if type == 'authority':
for site in sites:
site_hrn = interface_hrn + "." + site['login_base']
- if site_hrn == record_hrn and site['site_id'] == record['pointer']:
+ if site_hrn == record_hrn and site['site_id'] == record.pointer:
found = True
break
alt_username = person['email'].split("@")[0].replace(".", "_").replace("+", "_")
if username in [tmp_username, alt_username] and \
site['site_id'] in person['site_ids'] and \
- person['person_id'] == record['pointer']:
+ person['person_id'] == record.pointer:
found = True
break
slicename = hrn_to_pl_slicename(record_hrn)
for slice in slices:
if slicename == slice['name'] and \
- slice['slice_id'] == record['pointer']:
+ slice['slice_id'] == record.pointer:
found = True
break
tmp_nodename = node['hostname']
if tmp_nodename == nodename and \
node['site_id'] == site['site_id'] and \
- node['node_id'] == record['pointer']:
+ node['node_id'] == record.pointer:
found = True
break
else:
if not found:
try:
record_object = existing_records[(record_hrn, type)]
- record = SfaRecord(dict=record_object)
- logger.info("Import: deleting record: %s" % record.summary_string())
- record.delete()
+ logger.info("Import: deleting record: %s" % record)
+ dbsession.delete(record_object)
+ dbsession.commit()
except:
logger.log_exc("Import: failded to delete record")
# save pub keys
from sfa.util.sfalogging import logger
-from sfa.storage.table import SfaTable
+from sfa.storage.alchemy import dbsession
+from sfa.storage.persistentobjs import init_tables,drop_tables
def main():
- usage="%prog: trash the registry DB (the 'sfa' table in the 'planetlab5' database)"
+ usage="%prog: trash the registry DB"
parser = OptionParser(usage=usage)
- parser.add_option('-f','--file-system',dest='clean_fs',action='store_true',default=False,
- help='Clean up the /var/lib/sfa/authorities area as well')
- parser.add_option('-c','--certs',dest='clean_certs',action='store_true',default=False,
- help='Remove all cached certs/gids found in /var/lib/sfa/authorities area as well')
+ parser.add_option("-f","--file-system",dest='clean_fs',action='store_true',default=False,
+ help="Clean up the /var/lib/sfa/authorities area as well")
+ parser.add_option("-c","--certs",dest='clean_certs',action='store_true',default=False,
+ help="Remove all cached certs/gids found in /var/lib/sfa/authorities area as well")
+ parser.add_option("-0","--no-reinit",dest='reinit',action='store_false',default=True,
+ help="Do not reinitialize the database schema")
(options,args)=parser.parse_args()
if args:
parser.print_help()
sys.exit(1)
logger.info("Purging SFA records from database")
- table = SfaTable()
- table.nuke()
+ drop_tables(dbsession)
+ # for convenience we re-create the schema here, so there's no need for an explicit
+ # service sfa restart
+ # however in some (upgrade) scenarios this might be wrong
+ if options.reinit:
+ logger.info("re-creating empty schema")
+ init_tables(dbsession)
if options.clean_certs:
# remove the server certificate and all gids found in /var/lib/sfa/authorities
from sfa.trust.trustedroots import TrustedRoots
from sfa.trust.hierarchy import Hierarchy
from sfa.trust.gid import create_uuid
-from sfa.storage.table import SfaTable
-from sfa.storage.record import SfaRecord
-from sfa.storage.table import SfaTable
-
+from sfa.storage.persistentobjs import RegRecord, RegAuthority, RegUser
+from sfa.storage.alchemy import dbsession
def _un_unicode(str):
if isinstance(str, unicode):
def __init__(self):
self.logger = _SfaLogger(logfile='/var/log/sfa_import.log', loggername='importlog')
self.AuthHierarchy = Hierarchy()
- self.table = SfaTable()
self.config = Config()
self.TrustedRoots = TrustedRoots(Config.get_trustedroots_dir(self.config))
self.root_auth = self.config.SFA_REGISTRY_ROOT_AUTH
# create interface records
self.logger.info("Import: creating interface records")
+# xxx authority+ turning off the creation of authority+*
+# in fact his is required - used in SfaApi._getCredentialRaw
+# that tries to locate 'authority+sa'
self.create_interface_records()
# add local root authority's cert to trusted list
if not parent_hrn == hrn:
self.create_top_level_auth_records(parent_hrn)
- # enxure key and cert exists:
+ # ensure key and cert exists:
self.AuthHierarchy.create_top_level_auth(hrn)
# create the db record if it doesnt already exist
auth_info = self.AuthHierarchy.get_auth_info(hrn)
- auth_record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=-1, authority=get_authority(hrn))
- self.logger.info("Import: importing %s " % auth_record.summary_string())
- auth_record.sync()
+ auth_record = RegAuthority()
+ auth_record.type='authority'
+ auth_record.hrn=hrn
+ auth_record.gid=auth_info.get_gid_object()
+ auth_record.authority=get_authority(hrn)
+ auth_record.just_created()
+ dbsession.add (auth_record)
+ dbsession.commit()
+ self.logger.info("Import: imported authority (parent) %s " % auth_record)
def create_sm_client_record(self):
"""
self.AuthHierarchy.create_auth(urn)
auth_info = self.AuthHierarchy.get_auth_info(hrn)
- record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), \
- type="user", pointer=-1, authority=get_authority(hrn))
- self.logger.info("Import: importing %s " % record.summary_string())
- record.sync()
+ user_record = RegUser()
+ user_record.type='user'
+ user_record.hrn=hrn
+ user_record.gid=auth_info.get_gid_object()
+ user_record.authority=get_authority(hrn)
+ user_record.just_created()
+ dbsession.add (user_record)
+ dbsession.commit()
+ self.logger.info("Import: importing user (slicemanager) %s " % user_record)
def create_interface_records(self):
"""
Create a record for each SFA interface
"""
# just create certs for all sfa interfaces even if they
- # arent enabled
+ # aren't enabled
hrn = self.config.SFA_INTERFACE_HRN
- interfaces = ['authority+sa', 'authority+am', 'authority+sm']
- table = SfaTable()
auth_info = self.AuthHierarchy.get_auth_info(hrn)
pkey = auth_info.get_pkey_object()
- for interface in interfaces:
- urn = hrn_to_urn(hrn, interface)
+ for type in [ 'authority+sa', 'authority+am', 'authority+sm', ]:
+ urn = hrn_to_urn(hrn, type)
gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
- interface_record = SfaRecord(hrn=hrn, type=interface, pointer=-1,
- gid = gid, authority=get_authority(hrn))
- self.logger.info("Import: importing %s " % interface_record.summary_string())
- interface_record.sync()
+ # xxx this should probably use a RegAuthority, or a to-be-defined RegPeer object
+ # but for now we have to preserve the authority+<> stuff
+ interface_record = RegAuthority()
+ interface_record.type=type
+ interface_record.hrn=hrn
+ interface_record.gid= gid
+ interface_record.authority=get_authority(hrn)
+ interface_record.just_created()
+ dbsession.add (interface_record)
+ dbsession.commit()
+ self.logger.info("Import: imported authority (%s) %s " % (type,interface_record))
def delete_record(self, hrn, type):
# delete the record
- table = SfaTable()
- record_list = table.find({'type': type, 'hrn': hrn})
- for record in record_list:
- self.logger.info("Import: removing record %s %s" % (type, hrn))
- table.remove(record)
+ for rec in dbsession.query(RegRecord).filter_by(type=type,hrn=hrn):
+ dbsession.delete(rec)
+ dbsession.commit()
'interface':'aggregate',
'sfa': 2,
'geni_api': 2,
- 'geni_api_versions': {'2': 'http://%s:%s' % (api.config.SFA_SM_HOST, api.config.SFA_SM_PORT)},
+ 'geni_api_versions': {'2': 'http://%s:%s' % (api.config.SFA_AGGREGATE_HOST, api.config.SFA_AGGREGATE_PORT)},
'hrn':xrn.get_hrn(),
'urn':xrn.get_urn(),
}
import types
-import time
# for get_key_from_incoming_ip
import tempfile
import os
from sfa.trust.certificate import Certificate, Keypair, convert_public_key
from sfa.trust.gid import create_uuid
-from sfa.storage.record import SfaRecord
-from sfa.storage.table import SfaTable
+from sfa.storage.persistentobjs import make_record,RegRecord
+from sfa.storage.alchemy import dbsession
class RegistryManager:
if hrn != api.hrn])
xrn=Xrn(api.hrn)
return version_core({'interface':'registry',
+ 'sfa': 2,
+ 'geni_api': 2,
'hrn':xrn.get_hrn(),
'urn':xrn.get_urn(),
'peers':peers})
auth_hrn = api.auth.get_authority(hrn)
if not auth_hrn or hrn == api.config.SFA_INTERFACE_HRN:
auth_hrn = hrn
- # get record info
auth_info = api.auth.get_auth_info(auth_hrn)
- table = SfaTable()
- records = table.findObjects({'type': type, 'hrn': hrn})
- if not records:
- raise RecordNotFound(hrn)
- record = records[0]
+ # get record info
+ record=dbsession.query(RegRecord).filter_by(type=type,hrn=hrn).first()
+ if not record:
+ raise RecordNotFound("hrn=%s, type=%s"%(hrn,type))
# verify_cancreate_credential requires that the member lists
# (researchers, pis, etc) be filled in
- self.driver.augment_records_with_testbed_info (record)
- if not self.driver.is_enabled (record):
- raise AccountNotEnabled(": PlanetLab account %s is not enabled. Please contact your site PI" %(record['email']))
+ logger.debug("get credential before augment dict, keys=%s"%record.__dict__.keys())
+ self.driver.augment_records_with_testbed_info (record.__dict__)
+ logger.debug("get credential after augment dict, keys=%s"%record.__dict__.keys())
+ if not self.driver.is_enabled (record.__dict__):
+ raise AccountNotEnabled(": PlanetLab account %s is not enabled. Please contact your site PI" %(record.email))
# get the callers gid
# if this is a self cred the record's gid is the caller's gid
caller_hrn = caller_gid.get_hrn()
object_hrn = record.get_gid_object().get_hrn()
- rights = api.auth.determine_user_rights(caller_hrn, record)
+ rights = api.auth.determine_user_rights(caller_hrn, record.__dict__)
# make sure caller has rights to this object
if rights.is_empty():
- raise PermissionError(caller_hrn + " has no rights to " + record['name'])
+ raise PermissionError(caller_hrn + " has no rights to " + record.hrn)
- object_gid = GID(string=record['gid'])
+ object_gid = GID(string=record.gid)
new_cred = Credential(subject = object_gid.get_subject())
new_cred.set_gid_caller(caller_gid)
new_cred.set_gid_object(object_gid)
#new_cred.set_pubkey(object_gid.get_pubkey())
new_cred.set_privileges(rights)
new_cred.get_privileges().delegate_all_privileges(True)
- if 'expires' in record:
- date = utcparse(record['expires'])
+ if hasattr(record,'expires'):
+ date = utcparse(record.expires)
expires = datetime_to_epoch(date)
new_cred.set_expiration(int(expires))
auth_kind = "authority,ma,sa"
if not type:
type = Xrn(xrns).get_type()
hrns = [urn_to_hrn(xrn)[0] for xrn in xrns]
+
# load all known registry names into a prefix tree and attempt to find
# the longest matching prefix
- # create a dict where key is a registry hrn and its value is a
- # hrns at that registry (determined by the known prefix tree).
+ # create a dict where key is a registry hrn and its value is a list
+ # of hrns at that registry (determined by the known prefix tree).
xrn_dict = {}
registries = api.registries
tree = prefixTree()
credential = api.getCredential()
interface = api.registries[registry_hrn]
server_proxy = api.server_proxy(interface, credential)
- peer_records = server_proxy.Resolve(xrns, credential)
- records.extend([SfaRecord(dict=record).as_dict() for record in peer_records])
+ peer_records = server_proxy.Resolve(xrns, credential,type)
+ # pass foreign records as-is
+ # previous code used to read
+ # records.extend([SfaRecord(dict=record).as_dict() for record in peer_records])
+ # not sure why the records coming through xmlrpc had to be processed at all
+ records.extend(peer_records)
# try resolving the remaining unfound records at the local registry
local_hrns = list ( set(hrns).difference([record['hrn'] for record in records]) )
#
- table = SfaTable()
- local_records = table.findObjects({'hrn': local_hrns})
+ local_records = dbsession.query(RegRecord).filter(RegRecord.hrn.in_(local_hrns))
+ if type:
+ local_records = local_records.filter_by(type=type)
+ local_records=local_records.all()
+ logger.info("Resolve: local_records=%s (type=%s)"%(local_records,type))
+ local_dicts = [ record.__dict__ for record in local_records ]
if full:
# in full mode we get as much info as we can, which involves contacting the
# testbed for getting implementation details about the record
- self.driver.augment_records_with_testbed_info(local_records)
+ self.driver.augment_records_with_testbed_info(local_dicts)
# also we fill the 'url' field for known authorities
# used to be in the driver code, sounds like a poorman thing though
def solve_neighbour_url (record):
- if not record['type'].startswith('authority'): return
- hrn=record['hrn']
+ if not record.type.startswith('authority'): return
+ hrn=record.hrn
for neighbour_dict in [ api.aggregates, api.registries ]:
if hrn in neighbour_dict:
- record['url']=neighbour_dict[hrn].get_url()
+ record.url=neighbour_dict[hrn].get_url()
return
- [ solve_neighbour_url (record) for record in local_records ]
-
+ for record in local_records: solve_neighbour_url (record)
-
- # convert local record objects to dicts
- records.extend([dict(record) for record in local_records])
- if type:
- records = filter(lambda rec: rec['type'] in [type], records)
-
+ # convert local record objects to dicts for xmlrpc
+ # xxx somehow here calling dict(record) issues a weird error
+ # however record.todict() seems to work fine
+ # records.extend( [ dict(record) for record in local_records ] )
+ records.extend( [ record.todict() for record in local_records ] )
if not records:
raise RecordNotFound(str(hrns))
return records
- def List(self, api, xrn, origin_hrn=None):
+ def List (self, api, xrn, origin_hrn=None):
hrn, type = urn_to_hrn(xrn)
# load all know registry names into a prefix tree and attempt to find
# the longest matching prefix
- records = []
registries = api.registries
registry_hrns = registries.keys()
tree = prefixTree()
raise MissingAuthority(xrn)
# if the best match (longest matching hrn) is not the local registry,
# forward the request
- records = []
+ record_dicts = []
if registry_hrn != api.hrn:
credential = api.getCredential()
interface = api.registries[registry_hrn]
server_proxy = api.server_proxy(interface, credential)
record_list = server_proxy.List(xrn, credential)
- records = [SfaRecord(dict=record).as_dict() for record in record_list]
+ # same as above, no need to process what comes from through xmlrpc
+ # pass foreign records as-is
+ record_dicts = record_list
# if we still have not found the record yet, try the local registry
- if not records:
+ if not record_dicts:
if not api.auth.hierarchy.auth_exists(hrn):
raise MissingAuthority(hrn)
+ records = dbsession.query(RegRecord).filter_by(authority=hrn)
+ record_dicts=[ record.todict() for record in records ]
- table = SfaTable()
- records = table.find({'authority': hrn})
-
- return records
+ return record_dicts
def CreateGid(self, api, xrn, cert):
# subject_record describes the subject of the relationships
# ref_record contains the target values for the various relationships we need to manage
# (to begin with, this is just the slice x person relationship)
- def update_relations (self, subject_record, ref_record):
- type=subject_record['type']
+ def update_relations (self, subject_obj, ref_obj):
+ type=subject_obj.type
if type=='slice':
- self.update_relation(subject_record, 'researcher', ref_record.get('researcher'), 'user')
+ self.update_relation(subject_obj, 'researcher', ref_obj.researcher, 'user')
# field_key is the name of one field in the record, typically 'researcher' for a 'slice' record
# hrns is the list of hrns that should be linked to the subject from now on
# target_type would be e.g. 'user' in the 'slice' x 'researcher' example
- def update_relation (self, sfa_record, field_key, hrns, target_type):
+ def update_relation (self, record_obj, field_key, hrns, target_type):
# locate the linked objects in our db
- subject_type=sfa_record['type']
- subject_id=sfa_record['pointer']
- table = SfaTable()
- link_sfa_records = table.find ({'type':target_type, 'hrn': hrns})
- link_ids = [ rec.get('pointer') for rec in link_sfa_records ]
+ subject_type=record_obj.type
+ subject_id=record_obj.pointer
+ # get the 'pointer' field of all matching records
+ link_id_tuples = dbsession.query(RegRecord.pointer).filter_by(type=target_type).filter(RegRecord.hrn.in_(hrns)).all()
+ # sqlalchemy returns named tuples for columns
+ link_ids = [ tuple.pointer for tuple in link_id_tuples ]
self.driver.update_relation (subject_type, target_type, subject_id, link_ids)
-
- def Register(self, api, record):
+ def Register(self, api, record_dict):
- hrn, type = record['hrn'], record['type']
+ hrn, type = record_dict['hrn'], record_dict['type']
urn = hrn_to_urn(hrn,type)
# validate the type
if type not in ['authority', 'slice', 'node', 'user']:
raise UnknownSfaType(type)
- # check if record already exists
- table = SfaTable()
- existing_records = table.find({'type': type, 'hrn': hrn})
+ # check if record_dict already exists
+ existing_records = dbsession.query(RegRecord).filter_by(type=type,hrn=hrn).all()
if existing_records:
raise ExistingRecord(hrn)
- record = SfaRecord(dict = record)
- record['authority'] = get_authority(record['hrn'])
- auth_info = api.auth.get_auth_info(record['authority'])
+ assert ('type' in record_dict)
+ # returns the right type of RegRecord according to type in record
+ record = make_record(dict=record_dict)
+ record.just_created()
+ record.authority = get_authority(record.hrn)
+ auth_info = api.auth.get_auth_info(record.authority)
pub_key = None
# make sure record has a gid
- if 'gid' not in record:
+ if not record.gid:
uuid = create_uuid()
pkey = Keypair(create=True)
- if 'keys' in record and record['keys']:
- pub_key=record['keys']
+ if getattr(record,'keys',None):
+ pub_key=record.keys
# use only first key in record
- if isinstance(record['keys'], types.ListType):
- pub_key = record['keys'][0]
+ if isinstance(record.keys, types.ListType):
+ pub_key = record.keys[0]
pkey = convert_public_key(pub_key)
gid_object = api.auth.hierarchy.create_gid(urn, uuid, pkey)
gid = gid_object.save_to_string(save_parents=True)
- record['gid'] = gid
- record.set_gid(gid)
+ record.gid = gid
if type in ["authority"]:
# update the tree
# get the GID from the newly created authority
gid = auth_info.get_gid_object()
- record.set_gid(gid.save_to_string(save_parents=True))
+ record.gid=gid.save_to_string(save_parents=True)
# update testbed-specific data if needed
- pointer = self.driver.register (record, hrn, pub_key)
+ pointer = self.driver.register (record.__dict__, hrn, pub_key)
- record.set_pointer(pointer)
- record_id = table.insert(record)
- record['record_id'] = record_id
+ record.pointer=pointer
+ dbsession.add(record)
+ dbsession.commit()
# update membership for researchers, pis, owners, operators
self.update_relations (record, record)
return record.get_gid_object().save_to_string(save_parents=True)
def Update(self, api, record_dict):
- new_record = SfaRecord(dict = record_dict)
- type = new_record['type']
- hrn = new_record['hrn']
- urn = hrn_to_urn(hrn,type)
- table = SfaTable()
+ assert ('type' in record_dict)
+ new_record=RegRecord(dict=record_dict)
+ type = new_record.type
+ hrn = new_record.hrn
+
# make sure the record exists
- records = table.findObjects({'type': type, 'hrn': hrn})
- if not records:
- raise RecordNotFound(hrn)
- record = records[0]
- record['last_updated'] = time.gmtime()
+ record = dbsession.query(RegRecord).filter_by(type=type,hrn=hrn).first()
+ if not record:
+ raise RecordNotFound("hrn=%s, type=%s"%(hrn,type))
+ record.just_updated()
# validate the type
if type not in ['authority', 'slice', 'node', 'user']:
# Use the pointer from the existing record, not the one that the user
# gave us. This prevents the user from inserting a forged pointer
- pointer = record['pointer']
+ pointer = record.pointer
# is the a change in keys ?
new_key=None
if type=='user':
- if 'keys' in new_record and new_record['keys']:
- new_key=new_record['keys']
+ if getattr(new_key,'keys',None):
+ new_key=new_record.keys
if isinstance (new_key,types.ListType):
new_key=new_key[0]
# update the PLC information that was specified with the record
- if not self.driver.update (record, new_record, hrn, new_key):
+ if not self.driver.update (record.__dict__, new_record.__dict__, hrn, new_key):
logger.warning("driver.update failed")
# take new_key into account
# update the openssl key and gid
pkey = convert_public_key(new_key)
uuid = create_uuid()
+ urn = hrn_to_urn(hrn,type)
gid_object = api.auth.hierarchy.create_gid(urn, uuid, pkey)
gid = gid_object.save_to_string(save_parents=True)
- record['gid'] = gid
- record = SfaRecord(dict=record)
- table.update(record)
+ record.gid = gid
+ dsession.commit()
# update membership for researchers, pis, owners, operators
self.update_relations (record, new_record)
# expecting an Xrn instance
def Remove(self, api, xrn, origin_hrn=None):
-
- table = SfaTable()
- filter = {'hrn': xrn.get_hrn()}
hrn=xrn.get_hrn()
type=xrn.get_type()
+ request=dbsession.query(RegRecord).filter_by(hrn=hrn)
if type and type not in ['all', '*']:
- filter['type'] = type
+ request=request.filter_by(type=type)
- records = table.find(filter)
- if not records: raise RecordNotFound(hrn)
- record = records[0]
- type = record['type']
-
+ record = request.first()
+ if not record:
+ msg="Could not find hrn %s"%hrn
+ if type: msg += " type=%s"%type
+ raise RecordNotFound(msg)
+
+ type = record.type
if type not in ['slice', 'user', 'node', 'authority'] :
raise UnknownSfaType(type)
# call testbed callback first
# IIUC this is done on the local testbed TOO because of the refreshpeer link
- if not self.driver.remove(record):
+ if not self.driver.remove(record.__dict__):
logger.warning("driver.remove failed")
# delete from sfa db
- table.remove(record)
+ dbsession.delete(record)
+ dbsession.commit()
return 1
- # This is a PLC-specific thing...
+ # This is a PLC-specific thing, won't work with other platforms
def get_key_from_incoming_ip (self, api):
# verify that the callers's ip address exist in the db and is an interface
# for a node in the db
node = nodes[0]
# look up the sfa record
- table = SfaTable()
- records = table.findObjects({'type': 'node', 'pointer': node['node_id']})
- if not records:
- raise RecordNotFound("pointer:" + str(node['node_id']))
- record = records[0]
+ record=dbsession.query(RegRecord).filter_by(type='node',pointer=node['node_id']).first()
+ if not record:
+ raise RecordNotFound("node with pointer %s"%node['node_id'])
# generate a new keypair and gid
uuid = create_uuid()
pkey = Keypair(create=True)
- urn = hrn_to_urn(record['hrn'], record['type'])
+ urn = hrn_to_urn(record.hrn, record.type)
gid_object = api.auth.hierarchy.create_gid(urn, uuid, pkey)
gid = gid_object.save_to_string(save_parents=True)
- record['gid'] = gid
- record.set_gid(gid)
+ record.gid = gid
# update the record
- table.update(record)
+ dbsession.commit()
# attempt the scp the key
# and gid onto the node
--- /dev/null
+import types
+# for get_key_from_incoming_ip
+import tempfile
+import os
+import commands
+
+from sfa.util.faults import RecordNotFound, AccountNotEnabled, PermissionError, MissingAuthority, \
+ UnknownSfaType, ExistingRecord, NonExistingRecord
+from sfa.util.sfatime import utcparse, datetime_to_epoch
+from sfa.util.prefixTree import prefixTree
+from sfa.util.xrn import Xrn, get_authority, hrn_to_urn, urn_to_hrn
+from sfa.util.plxrn import hrn_to_pl_login_base
+from sfa.util.version import version_core
+from sfa.util.sfalogging import logger
+
+from sfa.trust.gid import GID
+from sfa.trust.credential import Credential
+from sfa.trust.certificate import Certificate, Keypair, convert_public_key
+from sfa.trust.gid import create_uuid
+
+from sfa.storage.persistentobjs import make_record,RegRecord
+from sfa.storage.alchemy import dbsession
+
+from sfa.managers.registry_manager import RegistryManager
+
+class RegistryManager(RegistryManager):
+
+ def GetCredential(self, api, xrn, type, is_self=False):
+ # convert xrn to hrn
+ if type:
+ hrn = urn_to_hrn(xrn)[0]
+ else:
+ hrn, type = urn_to_hrn(xrn)
+
+ # Is this a root or sub authority
+ auth_hrn = api.auth.get_authority(hrn)
+ if not auth_hrn or hrn == api.config.SFA_INTERFACE_HRN:
+ auth_hrn = hrn
+ auth_info = api.auth.get_auth_info(auth_hrn)
+ # get record info
+ record=dbsession.query(RegRecord).filter_by(type=type,hrn=hrn).first()
+ if not record:
+ raise RecordNotFound("hrn=%s, type=%s"%(hrn,type))
+
+ # verify_cancreate_credential requires that the member lists
+ # (researchers, pis, etc) be filled in
+ logger.debug("get credential before augment dict, keys=%s"%record.__dict__.keys())
+ self.driver.augment_records_with_testbed_info (record.__dict__)
+ logger.debug("get credential after augment dict, keys=%s"%record.__dict__.keys())
+ if not self.driver.is_enabled (record.__dict__):
+ raise AccountNotEnabled(": PlanetLab account %s is not enabled. Please contact your site PI" %(record.email))
+
+ # get the callers gid
+ # if this is a self cred the record's gid is the caller's gid
+ if is_self:
+ caller_hrn = hrn
+ caller_gid = record.get_gid_object()
+ else:
+ caller_gid = api.auth.client_cred.get_gid_caller()
+ caller_hrn = caller_gid.get_hrn()
+
+ object_hrn = record.get_gid_object().get_hrn()
+ rights = api.auth.determine_user_rights(caller_hrn, record.__dict__)
+ # make sure caller has rights to this object
+ if rights.is_empty():
+ raise PermissionError(caller_hrn + " has no rights to " + record.hrn)
+
+ object_gid = GID(string=record.gid)
+ new_cred = Credential(subject = object_gid.get_subject())
+ new_cred.set_gid_caller(caller_gid)
+ new_cred.set_gid_object(object_gid)
+ new_cred.set_issuer_keys(auth_info.get_privkey_filename(), auth_info.get_gid_filename())
+ #new_cred.set_pubkey(object_gid.get_pubkey())
+ new_cred.set_privileges(rights)
+ new_cred.get_privileges().delegate_all_privileges(True)
+ if hasattr(record,'expires'):
+ date = utcparse(record.expires)
+ expires = datetime_to_epoch(date)
+ new_cred.set_expiration(int(expires))
+ auth_kind = "authority,ma,sa"
+ # Parent not necessary, verify with certs
+ #new_cred.set_parent(api.auth.hierarchy.get_auth_cred(auth_hrn, kind=auth_kind))
+ new_cred.encode()
+ new_cred.sign()
+
+ return new_cred.save_to_string(save_parents=True)
+
+
+ # subject_record describes the subject of the relationships
+ # ref_record contains the target values for the various relationships we need to manage
+ # (to begin with, this is just the slice x person relationship)
+ def update_relations (self, subject_obj, ref_obj):
+ type=subject_obj.type
+ if type=='slice':
+ self.update_relation(subject_obj, 'researcher', ref_obj.researcher, 'user')
+
+ # field_key is the name of one field in the record, typically 'researcher' for a 'slice' record
+ # hrns is the list of hrns that should be linked to the subject from now on
+ # target_type would be e.g. 'user' in the 'slice' x 'researcher' example
+ def update_relation (self, record_obj, field_key, hrns, target_type):
+ # locate the linked objects in our db
+ subject_type=record_obj.type
+ subject_id=record_obj.pointer
+ # get the 'pointer' field of all matching records
+ link_id_tuples = dbsession.query(RegRecord.pointer).filter_by(type=target_type).filter(RegRecord.hrn.in_(hrns)).all()
+ # sqlalchemy returns named tuples for columns
+ link_ids = [ tuple.pointer for tuple in link_id_tuples ]
+ self.driver.update_relation (subject_type, target_type, subject_id, link_ids)
+
forward_options['rspec_version'] = version_manager.get_version('SFA 1').to_dict()
else:
forward_options['rspec_version'] = version_manager.get_version('ProtoGENI 2').to_dict()
+ forward_options['geni_rspec_version'] = {'type': 'geni', 'version': '3.0'}
rspec = server.ListResources(credential, forward_options)
return {"aggregate": aggregate, "rspec": rspec, "elapsed": time.time()-tStart, "status": "success"}
except Exception, e:
# look in cache first
cached_requested = options.get('cached', True)
- if not xrn and self.cache and cached_request:
+ if not xrn and self.cache and cached_requested:
rspec = self.cache.get(version_string)
if rspec:
api.logger.debug("SliceManager.ListResources returns cached advertisement")
call_id = options.get('call_id')
if Callids().already_handled(call_id): return True
- def _RenewSliver(server, xrn, creds, expiration_time, options):
- return server.RenewSliver(xrn, creds, expiration_time, options)
-
- (hrn, type) = urn_to_hrn(xrn)
+ def _RenewSliver(aggregate, server, xrn, creds, expiration_time, options):
+ try:
+ result=server.RenewSliver(xrn, creds, expiration_time, options)
+ if type(result)!=dict:
+ result = {"code": {"geni_code": 0}, value: result}
+ result["aggregate"] = aggregate
+ return result
+ except:
+ logger.log_exc('Something wrong in _RenewSliver with URL %s'%server.url)
+ return {"aggregate": aggregate, "exc_info": traceback.format_exc(), "code": {"geni_code": -1}, "value": False, "output": ""}
+
+ (hrn, urn_type) = urn_to_hrn(xrn)
# get the callers hrn
valid_cred = api.auth.checkCredentials(creds, 'renewsliver', hrn)[0]
caller_hrn = Credential(string=valid_cred).get_gid_caller().get_hrn()
-
+
# attempt to use delegated credential first
cred = api.getDelegatedCredential(creds)
if not cred:
- cred = api.getCredential()
+ cred = api.getCredential(minimumExpiration=31*86400)
threads = ThreadManager()
for aggregate in api.aggregates:
# prevent infinite loop. Dont send request back to caller
continue
interface = api.aggregates[aggregate]
server = api.server_proxy(interface, cred)
- threads.run(_RenewSliver, server, xrn, [cred], expiration_time, options)
- # 'and' the results
- results = [ReturnValue.get_value(result) for result in threads.get_results()]
- return reduce (lambda x,y: x and y, results , True)
-
+ threads.run(_RenewSliver, aggregate, server, xrn, [cred], expiration_time, options)
+
+ results = threads.get_results()
+
+ geni_code = 0
+ geni_output = ",".join([x.get("output","") for x in results])
+ geni_value = reduce (lambda x,y: x and y, [result.get("value",False) for result in results], True)
+ for agg_result in results:
+ agg_geni_code = agg_result["code"].get("geni_code",0)
+ if agg_geni_code:
+ geni_code = agg_geni_code
+
+ results = {"aggregates": results, "code": {"geni_code": geni_code}, "value": geni_value, "output": geni_output}
+
+ return results
+
def DeleteSliver(self, api, xrn, creds, options):
- call_id = options.get('call_id')
+ call_id = options.get('call_id')
if Callids().already_handled(call_id): return ""
def _DeleteSliver(server, xrn, creds, options):
# get the callers hrn
valid_cred = api.auth.checkCredentials(creds, 'deletesliver', hrn)[0]
caller_hrn = Credential(string=valid_cred).get_gid_caller().get_hrn()
-
+
# attempt to use delegated credential first
cred = api.getDelegatedCredential(creds)
if not cred:
# make sure users info is specified
if not users:
- msg = "'users' musst be specified and cannot be null. You may need to update your client."
+ msg = "'users' must be specified and cannot be null. You may need to update your client."
raise SfaInvalidArgument(name='users', extra=msg)
# flter rspec through sfatables
from sfa.trust.certificate import Certificate
from sfa.storage.parameter import Parameter, Mixed
-from sfa.storage.record import SfaRecord
class GetSelfCredential(Method):
"""
self.api.logger.info("interface: %s\tcaller-hrn: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, origin_hrn, hrn, self.name))
- # authenticate the gid
+ ### authenticate the gid
+ # import here so we can load this module at build-time for sfa2wsdl
+ #from sfa.storage.alchemy import dbsession
+ from sfa.storage.persistentobjs import RegRecord
+
+ # xxx-local - the current code runs Resolve, which would forward to
+ # another registry if needed
+ # I wonder if this is truly the intention, or shouldn't we instead
+ # only look in the local db ?
records = self.api.manager.Resolve(self.api, xrn, type)
if not records:
raise RecordNotFound(hrn)
- record = SfaRecord(dict=records[0])
- gid = record.get_gid_object()
+
+ record_obj = RegRecord (dict=records[0])
+ # xxx-local the local-only version would read
+ #record_obj = dbsession.query(RegRecord).filter_by(hrn=hrn).first()
+ #if not record_obj: raise RecordNotFound(hrn)
+ gid = record_obj.get_gid_object()
gid_str = gid.save_to_string(save_parents=True)
self.api.auth.authenticateGid(gid_str, [cert, type, hrn])
# authenticate the certificate against the gid in the db
from sfa.trust.credential import Credential
from sfa.storage.parameter import Parameter, Mixed
-from sfa.storage.record import SfaRecord
class List(Method):
"""
Parameter(type([str]), "List of credentials")),
]
- returns = [SfaRecord]
+ # xxx used to be [SfaRecord]
+ returns = [Parameter(dict, "registry record")]
def call(self, xrn, creds):
hrn, type = urn_to_hrn(xrn)
from sfa.trust.credential import Credential
from sfa.storage.parameter import Parameter, Mixed
-from sfa.storage.record import SfaRecord
class Resolve(Method):
"""
Parameter(list, "List of credentials)"))
]
- returns = [SfaRecord]
+ # xxx used to be [SfaRecord]
+ returns = [Parameter(dict, "registry record")]
def call(self, xrns, creds):
type = None
--- /dev/null
+import time
+import datetime
+#
+from sfa.util.faults import MissingSfaInfo, UnknownSfaType, \
+ RecordNotFound, SfaNotImplemented, SliverDoesNotExist
+
+from sfa.util.sfalogging import logger
+from sfa.util.defaultdict import defaultdict
+from sfa.util.sfatime import utcparse, datetime_to_string, datetime_to_epoch
+from sfa.util.xrn import Xrn, hrn_to_urn, get_leaf, urn_to_sliver_id
+from sfa.util.cache import Cache
+
+# one would think the driver should not need to mess with the SFA db, but..
+
+# used to be used in get_ticket
+#from sfa.trust.sfaticket import SfaTicket
+
+from sfa.rspecs.version_manager import VersionManager
+from sfa.rspecs.rspec import RSpec
+
+# the driver interface, mostly provides default behaviours
+from sfa.managers.driver import Driver
+
+from sfa.openstack.openstack_shell import OpenstackShell
+import sfa.plc.peers as peers
+from sfa.plc.plaggregate import PlAggregate
+from sfa.plc.plslices import PlSlices
+from sfa.util.plxrn import slicename_to_hrn, hostname_to_hrn, hrn_to_pl_slicename, hrn_to_pl_login_base
+
+
+def list_to_dict(recs, key):
+ """
+ convert a list of dictionaries into a dictionary keyed on the
+ specified dictionary key
+ """
+ return dict ( [ (rec[key],rec) for rec in recs ] )
+
+#
+# PlShell is just an xmlrpc serverproxy where methods
+# can be sent as-is; it takes care of authentication
+# from the global config
+#
+class OpenstackDriver (Driver):
+
+ # the cache instance is a class member so it survives across incoming requests
+ cache = None
+
+ def __init__ (self, config):
+ Driver.__init__ (self, config)
+ self.shell = OpenstackShell (config)
+ self.cache=None
+ if config.SFA_AGGREGATE_CACHING:
+ if OpenstackDriver.cache is None:
+ OpenstackDriver.cache = Cache()
+ self.cache = OpenstackDriver.cache
+
+ ########################################
+ ########## registry oriented
+ ########################################
+
+ ########## disabled users
+ def is_enabled (self, record):
+ # all records are enabled
+ return True
+
+ def augment_records_with_testbed_info (self, sfa_records):
+ return self.fill_record_info (sfa_records)
+
+ ##########
+ def register (self, sfa_record, hrn, pub_key):
+ type = sfa_record['type']
+ pl_record = self.sfa_fields_to_pl_fields(type, hrn, sfa_record)
+
+ if type == 'slice':
+ acceptable_fields=['url', 'instantiation', 'name', 'description']
+ # add slice description, name, researchers, PI
+ pass
+
+ elif type == 'user':
+ # add person roles, projects and keys
+ pass
+ return pointer
+
+ ##########
+ # xxx actually old_sfa_record comes filled with plc stuff as well in the original code
+ def update (self, old_sfa_record, new_sfa_record, hrn, new_key):
+ pointer = old_sfa_record['pointer']
+ type = old_sfa_record['type']
+
+ # new_key implemented for users only
+ if new_key and type not in [ 'user' ]:
+ raise UnknownSfaType(type)
+
+ elif type == "slice":
+ # can update description, researchers and PI
+ pass
+ elif type == "user":
+ # can update slices, keys and roles
+ pass
+ return True
+
+
+ ##########
+ def remove (self, sfa_record):
+ type=sfa_record['type']
+ name = Xrn(sfa_record['hrn']).get_leaf()
+ if type == 'user':
+ if self.shell.user_get(name):
+ self.shell.user_delete(name)
+ elif type == 'slice':
+ if self.shell.project_get(name):
+ self.shell.project_delete(name)
+ return True
+
+
+ ####################
+ def fill_record_info(self, records):
+ """
+ Given a (list of) SFA record, fill in the PLC specific
+ and SFA specific fields in the record.
+ """
+ if not isinstance(records, list):
+ records = [records]
+
+ for record in records:
+ name = Xrn(record['hrn']).get_leaf()
+ os_record = None
+ if record['type'] == 'user':
+ os_record = self.shell.user_get(name)
+ record['slices'] = [self.hrn + "." + proj.name for \
+ proj in os_record.projects]
+ record['roles'] = [role for role in os_record.roles]
+ keys = self.shell.key_pair_get_all_by_user(name)
+ record['keys'] = [key.public_key for key in keys]
+ elif record['type'] == 'slice':
+ os_record = self.shell.project_get(name)
+ record['description'] = os_record.description
+ record['PI'] = self.hrn + "." + os_record.project_manager
+ record['geni_creator'] = record['PI']
+ record['researcher'] = [self.hrn + "." + user.name for \
+ user in os_record.members]
+ else:
+ continue
+ record['geni_urn'] = hrn_to_urn(record['hrn'], record['type'])
+ record['geni_certificate'] = record['gid']
+ record['name'] = os_record.name
+ if os_record.created_at is not None:
+ record['date_created'] = datetime_to_string(utcparse(os_record.created_at))
+ if os_record.updated_at is not None:
+ record['last_updated'] = datetime_to_string(utcparse(os_record.updated_at))
+
+ return records
+
+
+ ####################
+ # plcapi works by changes, compute what needs to be added/deleted
+ def update_relation (self, subject_type, target_type, subject_id, target_ids):
+ # hard-wire the code for slice/user for now, could be smarter if needed
+ if subject_type =='slice' and target_type == 'user':
+ subject=self.shell.project_get(subject_id)[0]
+ current_target_ids = [user.name for user in subject.members]
+ add_target_ids = list ( set (target_ids).difference(current_target_ids))
+ del_target_ids = list ( set (current_target_ids).difference(target_ids))
+ logger.debug ("subject_id = %s (type=%s)"%(subject_id,type(subject_id)))
+ for target_id in add_target_ids:
+ self.shell.project_add_member(target_id,subject_id)
+ logger.debug ("add_target_id = %s (type=%s)"%(target_id,type(target_id)))
+ for target_id in del_target_ids:
+ logger.debug ("del_target_id = %s (type=%s)"%(target_id,type(target_id)))
+ self.shell.project_remove_member(target_id, subject_id)
+ else:
+ logger.info('unexpected relation to maintain, %s -> %s'%(subject_type,target_type))
+
+
+ ########################################
+ ########## aggregate oriented
+ ########################################
+
+ def testbed_name (self): return "openstack"
+
+ # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
+ def aggregate_version (self):
+ version_manager = VersionManager()
+ ad_rspec_versions = []
+ request_rspec_versions = []
+ for rspec_version in version_manager.versions:
+ if rspec_version.content_type in ['*', 'ad']:
+ ad_rspec_versions.append(rspec_version.to_dict())
+ if rspec_version.content_type in ['*', 'request']:
+ request_rspec_versions.append(rspec_version.to_dict())
+ return {
+ 'testbed':self.testbed_name(),
+ 'geni_request_rspec_versions': request_rspec_versions,
+ 'geni_ad_rspec_versions': ad_rspec_versions,
+ }
+
+ def list_slices (self, creds, options):
+ # look in cache first
+ if self.cache:
+ slices = self.cache.get('slices')
+ if slices:
+ logger.debug("PlDriver.list_slices returns from cache")
+ return slices
+
+ # get data from db
+ slices = self.shell.GetSlices({'peer_id': None}, ['name'])
+ slice_hrns = [slicename_to_hrn(self.hrn, slice['name']) for slice in slices]
+ slice_urns = [hrn_to_urn(slice_hrn, 'slice') for slice_hrn in slice_hrns]
+
+ # cache the result
+ if self.cache:
+ logger.debug ("PlDriver.list_slices stores value in cache")
+ self.cache.add('slices', slice_urns)
+
+ return slice_urns
+
+ # first 2 args are None in case of resource discovery
+ def list_resources (self, slice_urn, slice_hrn, creds, options):
+ cached_requested = options.get('cached', True)
+
+ version_manager = VersionManager()
+ # get the rspec's return format from options
+ rspec_version = version_manager.get_version(options.get('geni_rspec_version'))
+ version_string = "rspec_%s" % (rspec_version)
+
+ #panos adding the info option to the caching key (can be improved)
+ if options.get('info'):
+ version_string = version_string + "_"+options.get('info', 'default')
+
+ # look in cache first
+ if cached_requested and self.cache and not slice_hrn:
+ rspec = self.cache.get(version_string)
+ if rspec:
+ logger.debug("PlDriver.ListResources: returning cached advertisement")
+ return rspec
+
+ #panos: passing user-defined options
+ #print "manager options = ",options
+ aggregate = PlAggregate(self)
+ rspec = aggregate.get_rspec(slice_xrn=slice_urn, version=rspec_version,
+ options=options)
+
+ # cache the result
+ if self.cache and not slice_hrn:
+ logger.debug("PlDriver.ListResources: stores advertisement in cache")
+ self.cache.add(version_string, rspec)
+
+ return rspec
+
+ def sliver_status (self, slice_urn, slice_hrn):
+ # find out where this slice is currently running
+ slicename = hrn_to_pl_slicename(slice_hrn)
+
+ slices = self.shell.GetSlices([slicename], ['slice_id', 'node_ids','person_ids','name','expires'])
+ if len(slices) == 0:
+ raise SliverDoesNotExist("%s (used %s as slicename internally)" % (slice_hrn, slicename))
+ slice = slices[0]
+
+ # report about the local nodes only
+ nodes = self.shell.GetNodes({'node_id':slice['node_ids'],'peer_id':None},
+ ['node_id', 'hostname', 'site_id', 'boot_state', 'last_contact'])
+
+ if len(nodes) == 0:
+ raise SliverDoesNotExist("You have not allocated any slivers here")
+
+ site_ids = [node['site_id'] for node in nodes]
+
+ result = {}
+ top_level_status = 'unknown'
+ if nodes:
+ top_level_status = 'ready'
+ result['geni_urn'] = slice_urn
+ result['pl_login'] = slice['name']
+ result['pl_expires'] = datetime_to_string(utcparse(slice['expires']))
+
+ resources = []
+ for node in nodes:
+ res = {}
+ res['pl_hostname'] = node['hostname']
+ res['pl_boot_state'] = node['boot_state']
+ res['pl_last_contact'] = node['last_contact']
+ if node['last_contact'] is not None:
+
+ res['pl_last_contact'] = datetime_to_string(utcparse(node['last_contact']))
+ sliver_id = urn_to_sliver_id(slice_urn, slice['slice_id'], node['node_id'])
+ res['geni_urn'] = sliver_id
+ if node['boot_state'] == 'boot':
+ res['geni_status'] = 'ready'
+ else:
+ res['geni_status'] = 'failed'
+ top_level_status = 'failed'
+
+ res['geni_error'] = ''
+
+ resources.append(res)
+
+ result['geni_status'] = top_level_status
+ result['geni_resources'] = resources
+ return result
+
+ def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, users, options):
+
+ aggregate = PlAggregate(self)
+ slices = PlSlices(self)
+ peer = slices.get_peer(slice_hrn)
+ sfa_peer = slices.get_sfa_peer(slice_hrn)
+ slice_record=None
+ if users:
+ slice_record = users[0].get('slice_record', {})
+
+ # parse rspec
+ rspec = RSpec(rspec_string)
+ requested_attributes = rspec.version.get_slice_attributes()
+
+ # ensure site record exists
+ site = slices.verify_site(slice_hrn, slice_record, peer, sfa_peer, options=options)
+ # ensure slice record exists
+ slice = slices.verify_slice(slice_hrn, slice_record, peer, sfa_peer, options=options)
+ # ensure person records exists
+ persons = slices.verify_persons(slice_hrn, slice, users, peer, sfa_peer, options=options)
+ # ensure slice attributes exists
+ slices.verify_slice_attributes(slice, requested_attributes, options=options)
+
+ # add/remove slice from nodes
+ requested_slivers = [node.get('component_name') for node in rspec.version.get_nodes_with_slivers()]
+ nodes = slices.verify_slice_nodes(slice, requested_slivers, peer)
+
+ # add/remove links links
+ slices.verify_slice_links(slice, rspec.version.get_link_requests(), nodes)
+
+ # handle MyPLC peer association.
+ # only used by plc and ple.
+ slices.handle_peer(site, slice, persons, peer)
+
+ return aggregate.get_rspec(slice_xrn=slice_urn, version=rspec.version)
+
+ def delete_sliver (self, slice_urn, slice_hrn, creds, options):
+ slicename = hrn_to_pl_slicename(slice_hrn)
+ slices = self.shell.GetSlices({'name': slicename})
+ if not slices:
+ return 1
+ slice = slices[0]
+
+ # determine if this is a peer slice
+ # xxx I wonder if this would not need to use PlSlices.get_peer instead
+ # in which case plc.peers could be deprecated as this here
+ # is the only/last call to this last method in plc.peers
+ peer = peers.get_peer(self, slice_hrn)
+ try:
+ if peer:
+ self.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer)
+ self.shell.DeleteSliceFromNodes(slicename, slice['node_ids'])
+ finally:
+ if peer:
+ self.shell.BindObjectToPeer('slice', slice['slice_id'], peer, slice['peer_slice_id'])
+ return 1
+
+ def renew_sliver (self, slice_urn, slice_hrn, creds, expiration_time, options):
+ slicename = hrn_to_pl_slicename(slice_hrn)
+ slices = self.shell.GetSlices({'name': slicename}, ['slice_id'])
+ if not slices:
+ raise RecordNotFound(slice_hrn)
+ slice = slices[0]
+ requested_time = utcparse(expiration_time)
+ record = {'expires': int(datetime_to_epoch(requested_time))}
+ try:
+ self.shell.UpdateSlice(slice['slice_id'], record)
+ return True
+ except:
+ return False
+
+ # remove the 'enabled' tag
+ def start_slice (self, slice_urn, slice_hrn, creds):
+ slicename = hrn_to_pl_slicename(slice_hrn)
+ slices = self.shell.GetSlices({'name': slicename}, ['slice_id'])
+ if not slices:
+ raise RecordNotFound(slice_hrn)
+ slice_id = slices[0]['slice_id']
+ slice_tags = self.shell.GetSliceTags({'slice_id': slice_id, 'tagname': 'enabled'}, ['slice_tag_id'])
+ # just remove the tag if it exists
+ if slice_tags:
+ self.shell.DeleteSliceTag(slice_tags[0]['slice_tag_id'])
+ return 1
+
+ # set the 'enabled' tag to 0
+ def stop_slice (self, slice_urn, slice_hrn, creds):
+ slicename = hrn_to_pl_slicename(slice_hrn)
+ slices = self.shell.GetSlices({'name': slicename}, ['slice_id'])
+ if not slices:
+ raise RecordNotFound(slice_hrn)
+ slice_id = slices[0]['slice_id']
+ slice_tags = self.shell.GetSliceTags({'slice_id': slice_id, 'tagname': 'enabled'})
+ if not slice_tags:
+ self.shell.AddSliceTag(slice_id, 'enabled', '0')
+ elif slice_tags[0]['value'] != "0":
+ tag_id = slice_tags[0]['slice_tag_id']
+ self.shell.UpdateSliceTag(tag_id, '0')
+ return 1
+
+ def reset_slice (self, slice_urn, slice_hrn, creds):
+ raise SfaNotImplemented ("reset_slice not available at this interface")
+
+ # xxx this code is quite old and has not run for ages
+ # it is obviously totally broken and needs a rewrite
+ def get_ticket (self, slice_urn, slice_hrn, creds, rspec_string, options):
+ raise SfaNotImplemented,"PlDriver.get_ticket needs a rewrite"
+# please keep this code for future reference
+# slices = PlSlices(self)
+# peer = slices.get_peer(slice_hrn)
+# sfa_peer = slices.get_sfa_peer(slice_hrn)
+#
+# # get the slice record
+# credential = api.getCredential()
+# interface = api.registries[api.hrn]
+# registry = api.server_proxy(interface, credential)
+# records = registry.Resolve(xrn, credential)
+#
+# # make sure we get a local slice record
+# record = None
+# for tmp_record in records:
+# if tmp_record['type'] == 'slice' and \
+# not tmp_record['peer_authority']:
+# #Error (E0602, GetTicket): Undefined variable 'SliceRecord'
+# slice_record = SliceRecord(dict=tmp_record)
+# if not record:
+# raise RecordNotFound(slice_hrn)
+#
+# # similar to CreateSliver, we must verify that the required records exist
+# # at this aggregate before we can issue a ticket
+# # parse rspec
+# rspec = RSpec(rspec_string)
+# requested_attributes = rspec.version.get_slice_attributes()
+#
+# # ensure site record exists
+# site = slices.verify_site(slice_hrn, slice_record, peer, sfa_peer)
+# # ensure slice record exists
+# slice = slices.verify_slice(slice_hrn, slice_record, peer, sfa_peer)
+# # ensure person records exists
+# # xxx users is undefined in this context
+# persons = slices.verify_persons(slice_hrn, slice, users, peer, sfa_peer)
+# # ensure slice attributes exists
+# slices.verify_slice_attributes(slice, requested_attributes)
+#
+# # get sliver info
+# slivers = slices.get_slivers(slice_hrn)
+#
+# if not slivers:
+# raise SliverDoesNotExist(slice_hrn)
+#
+# # get initscripts
+# initscripts = []
+# data = {
+# 'timestamp': int(time.time()),
+# 'initscripts': initscripts,
+# 'slivers': slivers
+# }
+#
+# # create the ticket
+# object_gid = record.get_gid_object()
+# new_ticket = SfaTicket(subject = object_gid.get_subject())
+# new_ticket.set_gid_caller(api.auth.client_gid)
+# new_ticket.set_gid_object(object_gid)
+# new_ticket.set_issuer(key=api.key, subject=self.hrn)
+# new_ticket.set_pubkey(object_gid.get_pubkey())
+# new_ticket.set_attributes(data)
+# new_ticket.set_rspec(rspec)
+# #new_ticket.set_parent(api.auth.hierarchy.get_auth_ticket(auth_hrn))
+# new_ticket.encode()
+# new_ticket.sign()
+#
+# return new_ticket.save_to_string(save_parents=True)
--- /dev/null
+import sys
+import xmlrpclib
+import socket
+from urlparse import urlparse
+
+from sfa.util.sfalogging import logger
+
+class OpenstackShell:
+ """
+ A simple xmlrpc shell to a myplc instance
+ This class can receive all Openstack calls to the underlying testbed
+ """
+
+ # dont care about limiting calls yet
+ direct_calls = []
+ alias_calls = {}
+
+
+ # use the 'capability' auth mechanism for higher performance when the PLC db is local
+ def __init__ ( self, config ) :
+ url = config.SFA_PLC_URL
+ # try to figure if the url is local
+ hostname=urlparse(url).hostname
+ is_local=False
+ if hostname == 'localhost': is_local=True
+ # otherwise compare IP addresses;
+ # this might fail for any number of reasons, so let's harden that
+ try:
+ # xxx todo this seems to result in a DNS request for each incoming request to the AM
+ # should be cached or improved
+ url_ip=socket.gethostbyname(hostname)
+ local_ip=socket.gethostbyname(socket.gethostname())
+ if url_ip==local_ip: is_local=True
+ except:
+ pass
+
+
+ # Openstack provides a RESTful api but it is very limited, so we will
+ # ignore it for now and always use the native openstack (nova) library.
+ # This of course will not work if sfa is not installed on the same machine
+ # as the openstack-compute package.
+ if is_local:
+ try:
+ from nova.auth.manager import AuthManager, db, context
+ direct_access=True
+ except:
+ direct_access=False
+ if is_local and direct_access:
+
+ logger.debug('openstack access - native')
+ self.auth = context.get_admin_context()
+ # AuthManager isnt' really useful for much yet but it's
+ # more convenient to use than the db reference which requires
+ # a context. Lets hold onto the AuthManager reference for now.
+ #self.proxy = AuthManager()
+ self.auth_manager = AuthManager()
+ self.proxy = db
+
+ else:
+ self.auth = None
+ self.proxy = None
+ logger.debug('openstack access - REST')
+ raise SfaNotImplemented('openstack access - Rest')
+
+ def __getattr__(self, name):
+ def func(*args, **kwds):
+ result=getattr(self.proxy, name)(self.auth, *args, **kwds)
+ return result
+ return func
# most likely a default/global sliver attribute (node_id == None)
if tag['node_id'] not in slivers:
sliver = Sliver({'sliver_id': urn_to_sliver_id(slice_urn, slice['slice_id'], ""),
- 'name': 'plab-vserver',
+ 'name': slice['name'],
+ 'type': 'plab-vserver',
'tags': []})
slivers[tag['node_id']] = sliver
slivers[tag['node_id']]['tags'].append(tag)
rspec_node['slivers'] = [sliver]
# slivers always provide the ssh service
- login = Login({'authentication': 'ssh-keys', 'hostname': node['hostname'], 'port':'22'})
+ login = Login({'authentication': 'ssh-keys', 'hostname': node['hostname'], 'port':'22', 'username': sliver['name']})
service = Services({'login': login})
rspec_node['services'] = [service]
rspec_nodes.append(rspec_node)
from sfa.util.cache import Cache
# one would think the driver should not need to mess with the SFA db, but..
-from sfa.storage.table import SfaTable
+from sfa.storage.alchemy import dbsession
+from sfa.storage.persistentobjs import RegRecord
# used to be used in get_ticket
#from sfa.trust.sfaticket import SfaTicket
##
- # Convert SFA fields to PLC fields for use when registering up updating
+ # Convert SFA fields to PLC fields for use when registering or updating
# registry record in the PLC database
#
# we'll replace pl ids (person_ids) with hrns from the sfa records
# we obtain
- # get the sfa records
- table = SfaTable()
+ # get the registry records
person_list, persons = [], {}
- person_list = table.find({'type': 'user', 'pointer': person_ids})
+ person_list = dbsession.query (RegRecord).filter(RegRecord.pointer.in_(person_ids))
# create a hrns keyed on the sfa record's pointer.
# Its possible for multiple records to have the same pointer so
# the dict's value will be a list of hrns.
persons = defaultdict(list)
for person in person_list:
- persons[person['pointer']].append(person)
+ persons[person.pointer].append(person)
# get the pl records
pl_person_list, pl_persons = [], {}
# continue
sfa_info = {}
type = record['type']
+ logger.info("fill_record_sfa_info - incoming record typed %s"%type)
if (type == "slice"):
# all slice users are researchers
record['geni_urn'] = hrn_to_urn(record['hrn'], 'slice')
record['PI'] = []
record['researcher'] = []
for person_id in record.get('person_ids', []):
- hrns = [person['hrn'] for person in persons[person_id]]
+ hrns = [person.hrn for person in persons[person_id]]
record['researcher'].extend(hrns)
# pis at the slice's site
pl_pis = site_pis[record['site_id']]
pi_ids = [pi['person_id'] for pi in pl_pis]
for person_id in pi_ids:
- hrns = [person['hrn'] for person in persons[person_id]]
+ hrns = [person.hrn for person in persons[person_id]]
record['PI'].extend(hrns)
record['geni_creator'] = record['PI']
elif (type.startswith("authority")):
record['url'] = None
+ logger.info("fill_record_sfa_info - authority xherex")
if record['pointer'] != -1:
record['PI'] = []
record['operator'] = []
if pointer not in persons or pointer not in pl_persons:
# this means there is not sfa or pl record for this user
continue
- hrns = [person['hrn'] for person in persons[pointer]]
+ hrns = [person.hrn for person in persons[pointer]]
roles = pl_persons[pointer]['roles']
if 'pi' in roles:
record['PI'].extend(hrns)
# xxx TODO: URI, LatLong, IP, DNS
elif (type == "user"):
+ logger.info('setting user.email')
sfa_info['email'] = record.get("email", "")
sfa_info['geni_urn'] = hrn_to_urn(record['hrn'], 'user')
sfa_info['geni_certificate'] = record['gid']
from sfa.util.sfalogging import logger
from sfa.util.xrn import Xrn, get_leaf, get_authority, urn_to_hrn
#from sfa.util.policy import Policy
-from sfa.util.xrn import Xrn
+from sfa.util.plxrn import PlXrn
from sfa.rspecs.rspec import RSpec
from sfa.plc.vlink import VLink
from sfa.util.plxrn import hrn_to_pl_slicename
# unbind from peer so we can modify if necessary. Will bind back later
self.driver.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname'])
#Update existing record (e.g. expires field) it with the latest info.
- requested_expires = int(datetime_to_epoch(utcparse(slice_record['expires'])))
- if requested_expires and slice['expires'] != requested_expires:
- self.driver.shell.UpdateSlice( slice['slice_id'], {'expires' : requested_expires})
+ if slice_record.get('expires'):
+ requested_expires = int(datetime_to_epoch(utcparse(slice_record['expires'])))
+ if requested_expires and slice['expires'] != requested_expires:
+ self.driver.shell.UpdateSlice( slice['slice_id'], {'expires' : requested_expires})
return slice
for user in users:
hrn, type = urn_to_hrn(user['urn'])
username = get_leaf(hrn)
- login_base = get_leaf(get_authority(user['urn']))
+ login_base = PlXrn(xrn=user['urn']).pl_login_base()
user['username'] = username
user['site'] = login_base
self.driver.shell.DeleteSliceTag(attribute['slice_tag_id'])
except Exception, e:
logger.warn('Failed to remove sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
- % (name, value, node_id, str(e)))
+ % (slice['name'], attribute['value'], attribute.get('node_id'), str(e)))
# add requested_attributes
for attribute in added_slice_attributes:
self.driver.shell.AddSliceTag(slice['name'], attribute['name'], attribute['value'], attribute.get('node_id', None))
except Exception, e:
logger.warn('Failed to add sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
- % (name, value, node_id, str(e)))
+ % (slice['name'], attribute['value'], attribute.get('node_id'), str(e)))
fields = [
'authentication',
'hostname',
- 'port'
+ 'port',
+ 'username'
]
sliver_elem.set('name', sliver['type'])
if sliver.get('client_id'):
sliver_elem.set('client_id', sliver['client_id'])
- PGv2SliverType.add_sliver_attributes(sliver_elem, sliver.get('pl_tags', []))
+ PGv2SliverType.add_sliver_attributes(sliver_elem, sliver.get('tags', []))
@staticmethod
def add_sliver_attributes(xml, attributes):
- for attribute in attributes:
- if attribute['name'] == 'initscript':
- xml.add_element('{%s}initscript' % xml.namespaces['planetlab'], name=attribute['value'])
- elif tag['tagname'] == 'flack_info':
- attrib_elem = xml.add_element('{%s}info' % self.namespaces['flack'])
- attrib_dict = eval(tag['value'])
- for (key, value) in attrib_dict.items():
- attrib_elem.set(key, value)
+ if attributes:
+ for attribute in attributes:
+ if attribute['name'] == 'initscript':
+ xml.add_element('{%s}initscript' % xml.namespaces['planetlab'], name=attribute['value'])
+ elif tag['tagname'] == 'flack_info':
+ attrib_elem = xml.add_element('{%s}info' % self.namespaces['flack'])
+ attrib_dict = eval(tag['value'])
+ for (key, value) in attrib_dict.items():
+ attrib_elem.set(key, value)
@staticmethod
def get_slivers(xml, filter={}):
xpath = './default:sliver_type | ./sliver_type'
sliver_attributes = self.get_sliver_attributes(nodename, network)
for sliver_attribute in sliver_attributes:
sliver_attribute['node_id'] = nodename
- attributes.append(attribute)
+ attributes.append(sliver_attribute)
return attributes
def __init__(self, ip, port, key_file, cert_file):
SfaServer.__init__(self, ip, port, key_file, cert_file,'registry')
+ # initialize db schema
+ from sfa.storage.alchemy import dbsession
+ from sfa.storage.persistentobjs import init_tables
+ init_tables(dbsession)
#
# Registries is a dictionary of registry connections keyed on the registry hrn
from sfa.trust.gid import GID, create_uuid
from sfa.trust.hierarchy import Hierarchy
-from sfa.storage.table import SfaTable
+from sfa.storage.alchemy import dbsession
+from sfa.storage.persistentobjs import RegRecord
def main():
args = sys.argv
hrn = options.export
type = options.type
# check sfa table first
- filter = {'hrn': hrn}
- if type:
- filter['type'] = type
- table = SfaTable()
- records = table.find(filter)
- if not records:
+ request=dbsession.query(RegRecord).filter_by(hrn=hrn)
+ if type: request = request.filter_by(type=type)
+ record=request.first()
+ if not record:
# check the authorities hierarchy
hierarchy = Hierarchy()
try:
print "Record: %s not found" % hrn
sys.exit(1)
else:
- record = records[0]
- gid = GID(string=record['gid'])
+ gid = GID(string=record.gid)
# get the outfile
outfile = options.outfile
sys.exit(1)
# check if record exists in db
- table = SfaTable()
- records = table.find({'hrn': gid.get_hrn(), 'type': 'authority'})
- if not records:
+ record = dbsession.query(RegRecord).filter_by(type='authority',hrn=gid.get_hrn()).first()
+ if not record:
print "%s not found in record database" % gid.get_hrn()
sys.exit(1)
# update the database record
- record = records[0]
- record['gid'] = gid.save_to_string(save_parents=True)
- table.update(record)
+ record.gid = gid.save_to_string(save_parents=True)
+ dbsession.commit()
if options.verbose:
print "Imported %s gid into db" % record['hrn']
from sfa.trust.hierarchy import Hierarchy
from sfa.server.registry import Registries
-from sfa.storage.table import SfaTable
+from sfa.storage.alchemy import dbsession
+from sfa.storage.persistentobjs import RegRecord
from sfa.client.sfaserverproxy import SfaServerProxy
# Get the path to the sfa server key/cert files from
# the sfa hierarchy object
sfa_hierarchy = Hierarchy()
- sfa_key_path = sfa_hierarchy.basedir
- key_file = os.path.join(sfa_key_path, "server.key")
- cert_file = os.path.join(sfa_key_path, "server.cert")
+ auth_info = sfa_hierarchy.get_interface_auth_info()
+ key_file = auth_info.get_privkey_filename()
+ cert_file = auth_info.get_gid_filename()
key = Keypair(filename=key_file)
# get a connection to our local sfa registry
tree.load(registries.keys())
# get local peer records
- table = SfaTable()
- peer_records = table.find({'~peer_authority': None})
+ peer_records=dbsession.query(RegRecord).filter (RegRecord.peer_authority != None).all()
found_records = []
hrn_dict = {}
for record in peer_records:
- registry_hrn = tree.best_match(record['hrn'])
+ registry_hrn = tree.best_match(record.hrn)
if registry_hrn not in hrn_dict:
hrn_dict[registry_hrn] = []
- hrn_dict[registry_hrn].append(record['hrn'])
+ hrn_dict[registry_hrn].append(record.hrn)
# attempt to resolve the record at the authoritative interface
for registry_hrn in hrn_dict:
# remove what wasnt found
for peer_record in peer_records:
- if peer_record['hrn'] not in found_records:
- registries[sfa_api.hrn].Remove(peer_record['hrn'], credential, peer_record['type'])
+ if peer_record.hrn not in found_records:
+ registries[sfa_api.hrn].Remove(peer_record.hrn, credential, peer_record.type)
if __name__ == '__main__':
main()
from sfa.trust.certificate import Keypair, Certificate
from sfa.trust.hierarchy import Hierarchy
from sfa.trust.gid import GID
-
from sfa.server.sfaapi import SfaApi
from sfa.server.registry import Registries
from sfa.server.aggregate import Aggregates
+from sfa.client.return_value import ReturnValue
# after http://www.erlenstar.demon.co.uk/unix/faq_2.html
def daemon():
logger.info("get_trusted_certs: skipping non sfa aggregate: %s" % new_hrn)
continue
- trusted_gids = interface.get_trusted_certs()
+ trusted_gids = ReturnValue.get_value(interface.get_trusted_certs())
if trusted_gids:
# the gid we want should be the first one in the list,
# but lets make sure
message = "interface: %s\t" % (api.interface)
message += "unable to install trusted gid for %s" % \
(new_hrn)
- gid = GID(string=trusted_gids[0])
+ gid = GID(string=trusted_gid)
peer_gids.append(gid)
if gid.get_hrn() == new_hrn:
gid_filename = os.path.join(trusted_certs_dir, '%s.gid' % new_hrn)
Make sure there is a record in the registry for the specified gids.
Removes old records from the db.
"""
- # import SfaTable here so this module can be loaded by PlcComponentApi
- from sfa.storage.table import SfaTable
- from sfa.storage.record import SfaRecord
+ # import db stuff here here so this module can be loaded by PlcComponentApi
+ from sfa.storage.alchemy import dbsession
+ from sfa.storage.persistentobjs import RegRecord
if not gids:
return
- table = SfaTable()
# get records that actually exist in the db
gid_urns = [gid.get_urn() for gid in gids]
hrns_expected = [gid.get_hrn() for gid in gids]
- records_found = table.find({'hrn': hrns_expected, 'pointer': -1})
+ records_found = dbsession.query(RegRecord).\
+ filter_by(pointer=-1).filter(RegRecord.hrn.in_(hrns_expected)).all()
# remove old records
for record in records_found:
- if record['hrn'] not in hrns_expected and \
- record['hrn'] != self.api.config.SFA_INTERFACE_HRN:
- table.remove(record)
+ if record.hrn not in hrns_expected and \
+ record.hrn != self.api.config.SFA_INTERFACE_HRN:
+ dbsession.delete(record)
# TODO: store urn in the db so we do this in 1 query
for gid in gids:
hrn, type = gid.get_hrn(), gid.get_type()
- record = table.find({'hrn': hrn, 'type': type, 'pointer': -1})
+ record = dbsession.query(RegRecord).filter_by(hrn=hrn, type=type,pointer=-1).first()
if not record:
- record = {
- 'hrn': hrn, 'type': type, 'pointer': -1,
- 'authority': get_authority(hrn),
- 'gid': gid.save_to_string(save_parents=True),
- }
- record = SfaRecord(dict=record)
- table.insert(record)
+ record = RegRecord (dict= {'type':type,
+ 'hrn': hrn,
+ 'authority': get_authority(hrn),
+ 'gid': gid.save_to_string(save_parents=True),
+ })
+ dbsession.add(record)
+ dbsession.commit()
def main():
# Generate command line parser
import os, os.path
import datetime
-from sfa.util.faults import SfaFault, SfaAPIError
+from sfa.util.faults import SfaFault, SfaAPIError, RecordNotFound
from sfa.util.genicode import GENICODE
from sfa.util.config import Config
from sfa.util.cache import Cache
-from sfa.trust.auth import Auth
+from sfa.trust.auth import Auth
from sfa.trust.certificate import Keypair, Certificate
from sfa.trust.credential import Credential
from sfa.trust.rights import determine_rights
from sfa.client.return_value import ReturnValue
-# thgen xxx fixme this is wrong all right, but temporary, will use generic
-from sfa.storage.table import SfaTable
####################
class SfaApi (XmlrpcApi):
return server
- def getCredential(self):
+ def getCredential(self, minimumExpiration=0):
"""
- Return a valid credential for this interface.
+ Return a valid credential for this interface.
"""
type = 'authority'
path = self.config.SFA_DATA_DIR
cred = Credential(filename = cred_filename)
# make sure cred isnt expired
if not cred.get_expiration or \
- datetime.datetime.utcnow() < cred.get_expiration():
+ datetime.datetime.utcnow() + datetime.timedelta(seconds=minimumExpiration) < cred.get_expiration():
return cred.save_to_string(save_parents=True)
# get a new credential
if self.interface in ['registry']:
- cred = self.__getCredentialRaw()
+ cred = self._getCredentialRaw()
else:
- cred = self.__getCredential()
+ cred = self._getCredential()
cred.save_to_file(cred_filename, save_parents=True)
return cred.save_to_string(save_parents=True)
break
return delegated_cred
- def __getCredential(self):
+ def _getCredential(self):
"""
Get our credential from a remote registry
"""
cred = registry.GetCredential(self_cred, self.hrn, 'authority')
return Credential(string=cred)
- def __getCredentialRaw(self):
+ def _getCredentialRaw(self):
"""
Get our current credential directly from the local registry.
"""
if not auth_hrn or hrn == self.config.SFA_INTERFACE_HRN:
auth_hrn = hrn
auth_info = self.auth.get_auth_info(auth_hrn)
- # xxx thgen fixme - use SfaTable hardwired for now
- #table = self.SfaTable()
- table = SfaTable()
- records = table.findObjects({'hrn': hrn, 'type': 'authority+sa'})
- if not records:
- raise RecordNotFound
- record = records[0]
- type = record['type']
+ from sfa.storage.alchemy import dbsession
+ from sfa.storage.persistentobjs import RegRecord
+ record = dbsession.query(RegRecord).filter_by(type='authority+sa', hrn=hrn).first()
+ if not record:
+ raise RecordNotFound(hrn)
+ type = record.type
object_gid = record.get_gid_object()
new_cred = Credential(subject = object_gid.get_subject())
new_cred.set_gid_caller(object_gid)
def get_geni_code(self, result):
code = {
- 'geni_api': 2,
'geni_code': GENICODE.SUCCESS,
'am_type': 'sfa',
'am_code': None,
def prepare_response_v2_am(self, result):
response = {
+ 'geni_api': 2,
'code': self.get_geni_code(result),
'value': self.get_geni_value(result),
'output': self.get_geni_output(result),
+++ /dev/null
-#
-# PostgreSQL database interface. Sort of like DBI(3) (Database
-# independent interface for Perl).
-#
-#
-
-import re
-import traceback
-import commands
-from pprint import pformat
-from types import StringTypes, NoneType
-
-import psycopg2
-import psycopg2.extensions
-psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
-# UNICODEARRAY not exported yet
-psycopg2.extensions.register_type(psycopg2._psycopg.UNICODEARRAY)
-
-# allow to run sfa2wsdl if this is missing (for mac)
-import sys
-try: import pgdb
-except: print >> sys.stderr, "WARNING, could not import pgdb"
-
-from sfa.util.faults import SfaDBError
-from sfa.util.sfalogging import logger
-
-if not psycopg2:
- is8bit = re.compile("[\x80-\xff]").search
-
- def unicast(typecast):
- """
- pgdb returns raw UTF-8 strings. This function casts strings that
- apppear to contain non-ASCII characters to unicode objects.
- """
-
- def wrapper(*args, **kwds):
- value = typecast(*args, **kwds)
-
- # pgdb always encodes unicode objects as UTF-8 regardless of
- # the DB encoding (and gives you no option for overriding
- # the encoding), so always decode 8-bit objects as UTF-8.
- if isinstance(value, str) and is8bit(value):
- value = unicode(value, "utf-8")
-
- return value
-
- return wrapper
-
- pgdb.pgdbTypeCache.typecast = unicast(pgdb.pgdbTypeCache.typecast)
-
-def handle_exception(f):
- def wrapper(*args, **kwds):
- try: return f(*args, **kwds)
- except Exception, fault:
- raise SfaDBError(str(fault))
- return wrapper
-
-class PostgreSQL:
- def __init__(self, config):
- self.config = config
- self.debug = False
-# self.debug = True
- self.connection = None
-
- @handle_exception
- def cursor(self):
- if self.connection is None:
- # (Re)initialize database connection
- if psycopg2:
- try:
- # Try UNIX socket first
- self.connection = psycopg2.connect(user = self.config.SFA_DB_USER,
- password = self.config.SFA_DB_PASSWORD,
- database = self.config.SFA_DB_NAME)
- except psycopg2.OperationalError:
- # Fall back on TCP
- self.connection = psycopg2.connect(user = self.config.SFA_DB_USER,
- password = self.config.SFA_DB_PASSWORD,
- database = self.config.SFA_DB_NAME,
- host = self.config.SFA_DB_HOST,
- port = self.config.SFA_DB_PORT)
- self.connection.set_client_encoding("UNICODE")
- else:
- self.connection = pgdb.connect(user = self.config.SFA_DB_USER,
- password = self.config.SFA_DB_PASSWORD,
- host = "%s:%d" % (self.config.SFA_DB_HOST, self.config.SFA_DB_PORT),
- database = self.config.SFA_DB_NAME)
-
- (self.rowcount, self.description, self.lastrowid) = \
- (None, None, None)
-
- return self.connection.cursor()
-
- def close(self):
- if self.connection is not None:
- self.connection.close()
- self.connection = None
-
- def quote(self, value):
- """
- Returns quoted version of the specified value.
- """
-
- # The pgdb._quote function is good enough for general SQL
- # quoting, except for array types.
- if isinstance(value, (list, tuple, set)):
- return "ARRAY[%s]" % ", ".join(map, self.quote, value)
- else:
- return pgdb._quote(value)
-
- quote = classmethod(quote)
-
- def param(self, name, value):
- # None is converted to the unquoted string NULL
- if isinstance(value, NoneType):
- conversion = "s"
- # True and False are also converted to unquoted strings
- elif isinstance(value, bool):
- conversion = "s"
- elif isinstance(value, float):
- conversion = "f"
- elif not isinstance(value, StringTypes):
- conversion = "d"
- else:
- conversion = "s"
-
- return '%(' + name + ')' + conversion
-
- param = classmethod(param)
-
- def begin_work(self):
- # Implicit in pgdb.connect()
- pass
-
- def commit(self):
- self.connection.commit()
-
- def rollback(self):
- self.connection.rollback()
-
- def do(self, query, params = None):
- cursor = self.execute(query, params)
- cursor.close()
- return self.rowcount
-
- def next_id(self, table_name, primary_key):
- sequence = "%(table_name)s_%(primary_key)s_seq" % locals()
- sql = "SELECT nextval('%(sequence)s')" % locals()
- rows = self.selectall(sql, hashref = False)
- if rows:
- return rows[0][0]
- return None
-
- def last_insert_id(self, table_name, primary_key):
- if isinstance(self.lastrowid, int):
- sql = "SELECT %s FROM %s WHERE oid = %d" % \
- (primary_key, table_name, self.lastrowid)
- rows = self.selectall(sql, hashref = False)
- if rows:
- return rows[0][0]
-
- return None
-
- # modified for psycopg2-2.0.7
- # executemany is undefined for SELECT's
- # see http://www.python.org/dev/peps/pep-0249/
- # accepts either None, a single dict, a tuple of single dict - in which case it execute's
- # or a tuple of several dicts, in which case it executemany's
- def execute(self, query, params = None):
-
- cursor = self.cursor()
- try:
-
- # psycopg2 requires %()s format for all parameters,
- # regardless of type.
- # this needs to be done carefully though as with pattern-based filters
- # we might have percents embedded in the query
- # so e.g. GetPersons({'email':'*fake*'}) was resulting in .. LIKE '%sake%'
- if psycopg2:
- query = re.sub(r'(%\([^)]*\)|%)[df]', r'\1s', query)
- # rewrite wildcards set by Filter.py as '***' into '%'
- query = query.replace ('***','%')
-
- if not params:
- if self.debug:
- logger.debug('execute0 %r'%query)
- cursor.execute(query)
- elif isinstance(params,dict):
- if self.debug:
- logger.debug('execute-dict: params=[%r] query=[%r]'%(params,query%params))
- cursor.execute(query,params)
- elif isinstance(params,tuple) and len(params)==1:
- if self.debug:
- logger.debug('execute-tuple %r'%(query%params[0]))
- cursor.execute(query,params[0])
- else:
- param_seq=(params,)
- if self.debug:
- for params in param_seq:
- logger.debug('executemany %r'%(query%params))
- cursor.executemany(query, param_seq)
- (self.rowcount, self.description, self.lastrowid) = \
- (cursor.rowcount, cursor.description, cursor.lastrowid)
- except Exception, e:
- try:
- self.rollback()
- except:
- pass
- uuid = commands.getoutput("uuidgen")
- logger.error("Database error %s:" % uuid)
- logger.error("Exception=%r"%e)
- logger.error("Query=%r"%query)
- logger.error("Params=%r"%pformat(params))
- logger.log_exc("PostgreSQL.execute caught exception")
- raise SfaDBError("Please contact support: %s" % str(e))
-
- return cursor
-
- def selectall(self, query, params = None, hashref = True, key_field = None):
- """
- Return each row as a dictionary keyed on field name (like DBI
- selectrow_hashref()). If key_field is specified, return rows
- as a dictionary keyed on the specified field (like DBI
- selectall_hashref()).
-
- If params is specified, the specified parameters will be bound
- to the query.
- """
-
- cursor = self.execute(query, params)
- rows = cursor.fetchall()
- cursor.close()
- self.commit()
- if hashref or key_field is not None:
- # Return each row as a dictionary keyed on field name
- # (like DBI selectrow_hashref()).
- labels = [column[0] for column in self.description]
- rows = [dict(zip(labels, row)) for row in rows]
-
- if key_field is not None and key_field in labels:
- # Return rows as a dictionary keyed on the specified field
- # (like DBI selectall_hashref()).
- return dict([(row[key_field], row) for row in rows])
- else:
- return rows
-
- def fields(self, table, notnull = None, hasdef = None):
- """
- Return the names of the fields of the specified table.
- """
-
- if hasattr(self, 'fields_cache'):
- if self.fields_cache.has_key((table, notnull, hasdef)):
- return self.fields_cache[(table, notnull, hasdef)]
- else:
- self.fields_cache = {}
-
- sql = "SELECT attname FROM pg_attribute, pg_class" \
- " WHERE pg_class.oid = attrelid" \
- " AND attnum > 0 AND relname = %(table)s"
-
- if notnull is not None:
- sql += " AND attnotnull is %(notnull)s"
-
- if hasdef is not None:
- sql += " AND atthasdef is %(hasdef)s"
-
- rows = self.selectall(sql, locals(), hashref = False)
-
- self.fields_cache[(table, notnull, hasdef)] = [row[0] for row in rows]
-
- return self.fields_cache[(table, notnull, hasdef)]
--- /dev/null
+from types import StringTypes
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from sqlalchemy import Column, Integer, String
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy import ForeignKey
+
+from sfa.util.sfalogging import logger
+
+# this module is designed to be loaded when the configured db server is reachable
+# OTOH persistentobjs can be loaded from anywhere including the client-side
+
+class Alchemy:
+
+ def __init__ (self, config):
+ dbname="sfa"
+ # will be created lazily on-demand
+ self._session = None
+ # the former PostgreSQL.py used the psycopg2 directly and was doing
+ #self.connection.set_client_encoding("UNICODE")
+ # it's unclear how to achieve this in sqlalchemy, nor if it's needed at all
+ # http://www.sqlalchemy.org/docs/dialects/postgresql.html#unicode
+ # we indeed have /var/lib/pgsql/data/postgresql.conf where
+ # this setting is unset, it might be an angle to tweak that if need be
+ # try a unix socket first - omitting the hostname does the trick
+ unix_desc = "postgresql+psycopg2://%s:%s@:%s/%s"%\
+ (config.SFA_DB_USER,config.SFA_DB_PASSWORD,config.SFA_DB_PORT,dbname)
+ # the TCP fallback method
+ tcp_desc = "postgresql+psycopg2://%s:%s@%s:%s/%s"%\
+ (config.SFA_DB_USER,config.SFA_DB_PASSWORD,config.SFA_DB_HOST,config.SFA_DB_PORT,dbname)
+ for engine_desc in [ unix_desc, tcp_desc ] :
+ try:
+ self.engine = create_engine (engine_desc)
+ self.check()
+ return
+ except:
+ pass
+ self.engine=None
+ raise Exception,"Could not connect to database"
+
+
+ # expects boolean True: debug is ON or False: debug is OFF
+ def debug (self, echo):
+ self.engine.echo=echo
+
+ def check (self):
+ self.engine.execute ("select 1").scalar()
+
+ def session (self):
+ if self._session is None:
+ Session=sessionmaker ()
+ self._session=Session(bind=self.engine)
+ return self._session
+
+ def close_session (self):
+ if self._session is None: return
+ self._session.close()
+ self._session=None
+
+####################
+from sfa.util.config import Config
+
+alchemy=Alchemy (Config())
+engine=alchemy.engine
+dbsession=alchemy.session()
+
+++ /dev/null
-from types import StringTypes
-import pgdb
-
-from sfa.util.faults import SfaInvalidArgument
-
-from sfa.storage.parameter import Parameter, Mixed, python_type
-
-class Filter(Parameter, dict):
- """
- A type of parameter that represents a filter on one or more
- columns of a database table.
- Special features provide support for negation, upper and lower bounds,
- as well as sorting and clipping.
-
-
- fields should be a dictionary of field names and types
- Only filters on non-sequence type fields are supported.
- example : fields = {'node_id': Parameter(int, "Node identifier"),
- 'hostname': Parameter(int, "Fully qualified hostname", max = 255),
- ...}
-
-
- filter should be a dictionary of field names and values
- representing the criteria for filtering.
- example : filter = { 'hostname' : '*.edu' , site_id : [34,54] }
- Whether the filter represents an intersection (AND) or a union (OR)
- of these criteria is determined by the join_with argument
- provided to the sql method below
-
- Special features:
-
- * a field starting with the ~ character means negation.
- example : filter = { '~peer_id' : None }
-
- * a field starting with < [ ] or > means lower than or greater than
- < > uses strict comparison
- [ ] is for using <= or >= instead
- example : filter = { ']event_id' : 2305 }
- example : filter = { '>time' : 1178531418 }
- in this example the integer value denotes a unix timestamp
-
- * if a value is a sequence type, then it should represent
- a list of possible values for that field
- example : filter = { 'node_id' : [12,34,56] }
-
- * a (string) value containing either a * or a % character is
- treated as a (sql) pattern; * are replaced with % that is the
- SQL wildcard character.
- example : filter = { 'hostname' : '*.jp' }
-
- * fields starting with - are special and relate to row selection, i.e. sorting and clipping
- * '-SORT' : a field name, or an ordered list of field names that are used for sorting
- these fields may start with + (default) or - for denoting increasing or decreasing order
- example : filter = { '-SORT' : [ '+node_id', '-hostname' ] }
- * '-OFFSET' : the number of first rows to be ommitted
- * '-LIMIT' : the amount of rows to be returned
- example : filter = { '-OFFSET' : 100, '-LIMIT':25}
-
- A realistic example would read
- GetNodes ( { 'node_type' : 'regular' , 'hostname' : '*.edu' , '-SORT' : 'hostname' , '-OFFSET' : 30 , '-LIMIT' : 25 } )
- and that would return regular (usual) nodes matching '*.edu' in alphabetical order from 31th to 55th
- """
-
- def __init__(self, fields = {}, filter = {}, doc = "Attribute filter"):
- # Store the filter in our dict instance
- valid_fields = {}
- for field in filter:
- if field in fields:
- valid_fields[field] = filter[field]
- dict.__init__(self, valid_fields)
-
- # Declare ourselves as a type of parameter that can take
- # either a value or a list of values for each of the specified
- # fields.
- self.fields = dict ( [ ( field, Mixed (expected, [expected]))
- for (field,expected) in fields.iteritems()
- if python_type(expected) not in (list, tuple, set) ] )
-
- # Null filter means no filter
- Parameter.__init__(self, self.fields, doc = doc, nullok = True)
-
- def quote(self, value):
- """
- Returns quoted version of the specified value.
- """
-
- # The pgdb._quote function is good enough for general SQL
- # quoting, except for array types.
- if isinstance(value, (list, tuple, set)):
- return "ARRAY[%s]" % ", ".join(map(self.quote, value))
- else:
- return pgdb._quote(value)
-
- def sql(self, join_with = "AND"):
- """
- Returns a SQL conditional that represents this filter.
- """
-
- # So that we always return something
- if join_with == "AND":
- conditionals = ["True"]
- elif join_with == "OR":
- conditionals = ["False"]
- else:
- assert join_with in ("AND", "OR")
-
- # init
- sorts = []
- clips = []
-
- for field, value in self.iteritems():
- # handle negation, numeric comparisons
- # simple, 1-depth only mechanism
-
- modifiers={'~' : False,
- '<' : False, '>' : False,
- '[' : False, ']' : False,
- '-' : False,
- }
-
- for char in modifiers.keys():
- if field[0] == char:
- modifiers[char]=True
- field = field[1:]
- break
-
- # filter on fields
- if not modifiers['-']:
- if field not in self.fields:
- raise SfaInvalidArgument, "Invalid filter field '%s'" % field
-
- if isinstance(value, (list, tuple, set)):
- # handling filters like '~slice_id':[]
- # this should return true, as it's the opposite of 'slice_id':[] which is false
- # prior to this fix, 'slice_id':[] would have returned ``slice_id IN (NULL) '' which is unknown
- # so it worked by coincidence, but the negation '~slice_ids':[] would return false too
- if not value:
- field=""
- operator=""
- value = "FALSE"
- else:
- operator = "IN"
- value = map(str, map(self.quote, value))
- value = "(%s)" % ", ".join(value)
- else:
- if value is None:
- operator = "IS"
- value = "NULL"
- elif isinstance(value, StringTypes) and \
- (value.find("*") > -1 or value.find("%") > -1):
- operator = "LIKE"
- # insert *** in pattern instead of either * or %
- # we dont use % as requests are likely to %-expansion later on
- # actual replacement to % done in PostgreSQL.py
- value = value.replace ('*','***')
- value = value.replace ('%','***')
- value = str(self.quote(value))
- else:
- operator = "="
- if modifiers['<']:
- operator='<'
- if modifiers['>']:
- operator='>'
- if modifiers['[']:
- operator='<='
- if modifiers[']']:
- operator='>='
- else:
- value = str(self.quote(value))
-
- clause = "%s %s %s" % (field, operator, value)
-
- if modifiers['~']:
- clause = " ( NOT %s ) " % (clause)
-
- conditionals.append(clause)
- # sorting and clipping
- else:
- if field not in ('SORT','OFFSET','LIMIT'):
- raise SfaInvalidArgument, "Invalid filter, unknown sort and clip field %r"%field
- # sorting
- if field == 'SORT':
- if not isinstance(value,(list,tuple,set)):
- value=[value]
- for field in value:
- order = 'ASC'
- if field[0] == '+':
- field = field[1:]
- elif field[0] == '-':
- field = field[1:]
- order = 'DESC'
- if field not in self.fields:
- raise SfaInvalidArgument, "Invalid field %r in SORT filter"%field
- sorts.append("%s %s"%(field,order))
- # clipping
- elif field == 'OFFSET':
- clips.append("OFFSET %d"%value)
- # clipping continued
- elif field == 'LIMIT' :
- clips.append("LIMIT %d"%value)
-
- where_part = (" %s " % join_with).join(conditionals)
- clip_part = ""
- if sorts:
- clip_part += " ORDER BY " + ",".join(sorts)
- if clips:
- clip_part += " " + " ".join(clips)
- return (where_part,clip_part)
+++ /dev/null
-DROP VIEW view_records;
-DROP VIEW record_keys;
-DROP TABLE record_key;
-DROP TABLE keys;
-DROP TABLE key_types;
-
-------------------------------------------------------------
-UPDATE sfa_db_version SET subversion = 1;
+++ /dev/null
-----
--- the purpose of this migration is to enrich the proper SFA table
--- so that the registry can perform reasonably in standalone mode,
--- i.e. without any underlying myplc
--- this is desirable in the perspective of providing a generic framework
--- for non myplc-based testbeds
--- prior to this change, the registry needed to inspect the myplc db in order
--- to retrieve keys and roles, so as to be able to make the right decisions in
--- terms of delivering credentials
-----
-
---------------------------------------------------------------------------------
--- Authentication Keys
---------------------------------------------------------------------------------
--- Valid key types
-CREATE TABLE key_types (
- key_type text PRIMARY KEY
-) WITH OIDS;
-INSERT INTO key_types (key_type) VALUES ('ssh');
-
--- Authentication keys
-CREATE TABLE keys (
- key_id serial PRIMARY KEY,
- key_type text REFERENCES key_types NOT NULL,
- key text NOT NULL
-) WITH OIDS;
-
--- attaching keys to records
-CREATE TABLE record_key (
- record_id integer REFERENCES records NOT NULL,
- key_id integer REFERENCES keys PRIMARY KEY
-) WITH OIDS;
-CREATE INDEX record_key_record_id_idx ON record_key (record_id);
-
--- get all keys attached to one record
-CREATE OR REPLACE VIEW record_keys AS
-SELECT record_id,
-array_accum(key_id) AS key_ids
-FROM record_key
-GROUP BY record_id;
-
--- a synthesis view for records
-CREATE OR REPLACE VIEW view_records AS
-SELECT
-records.record_id,
-records.hrn,
-records.authority,
-records.peer_authority,
-records.gid,
-records.type,
-records.pointer,
-records.date_created,
-records.last_updated,
-COALESCE((SELECT key_ids FROM record_keys WHERE record_keys.record_id = records.record_id), '{}') AS key_ids
-FROM records;
-
-------------------------------------------------------------
-UPDATE sfa_db_version SET subversion = 2;
--- /dev/null
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/
--- /dev/null
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=future migrations in the SFA registry database
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=sfa_db_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=['postgres']
--- /dev/null
+from types import StringTypes
+from datetime import datetime
+
+from sqlalchemy import Column, Integer, String, DateTime
+from sqlalchemy import Table, Column, MetaData, join, ForeignKey
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.orm import column_property
+from sqlalchemy.orm import object_mapper
+from sqlalchemy.orm import validates
+from sqlalchemy.ext.declarative import declarative_base
+
+from sfa.util.sfalogging import logger
+from sfa.util.xml import XML
+
+from sfa.trust.gid import GID
+
+##############################
+Base=declarative_base()
+
+####################
+# dicts vs objects
+####################
+# historically the front end to the db dealt with dicts, so the code was only dealing with dicts
+# sqlalchemy however offers an object interface, meaning that you write obj.id instead of obj['id']
+# which is admittedly much nicer
+# however we still need to deal with dictionaries if only for the xmlrpc layer
+#
+# here are a few utilities for this
+#
+# (*) first off, when an old pieve of code needs to be used as-is, if only temporarily, the simplest trick
+# is to use obj.__dict__
+# this behaves exactly like required, i.e. obj.__dict__['field']='new value' does change obj.field
+# however this depends on sqlalchemy's implementation so it should be avoided
+#
+# (*) second, when an object needs to be exposed to the xmlrpc layer, we need to convert it into a dict
+# remember though that writing the resulting dictionary won't change the object
+# essentially obj.__dict__ would be fine too, except that we want to discard alchemy private keys starting with '_'
+# 2 ways are provided for that:
+# . dict(obj)
+# . obj.todict()
+# the former dict(obj) relies on __iter__() and next() below, and does not rely on the fields names
+# although it seems to work fine, I've found cases where it issues a weird python error that I could not get right
+# so the latter obj.todict() seems more reliable but more hacky as is relies on the form of fields, so this can probably be improved
+#
+# (*) finally for converting a dictionary into an sqlalchemy object, we provide
+# obj.load_from_dict(dict)
+
+class AlchemyObj:
+ def __iter__(self):
+ self._i = iter(object_mapper(self).columns)
+ return self
+ def next(self):
+ n = self._i.next().name
+ return n, getattr(self, n)
+ def todict (self):
+ d=self.__dict__
+ keys=[k for k in d.keys() if not k.startswith('_')]
+ return dict ( [ (k,d[k]) for k in keys ] )
+ def load_from_dict (self, d):
+ for (k,v) in d.iteritems():
+ # experimental
+ if isinstance(v, StringTypes) and v.lower() in ['true']: v=True
+ if isinstance(v, StringTypes) and v.lower() in ['false']: v=False
+ setattr(self,k,v)
+
+ # in addition we provide convenience for converting to and from xml records
+ # for this purpose only, we need the subclasses to define 'fields' as either
+ # a list or a dictionary
+ def xml_fields (self):
+ fields=self.fields
+ if isinstance(fields,dict): fields=fields.keys()
+ return fields
+
+ def save_as_xml (self):
+ # xxx not sure about the scope here
+ input_dict = dict( [ (key, getattr(self.key), ) for key in self.xml_fields() if getattr(self,key,None) ] )
+ xml_record=XML("<record />")
+ xml_record.parse_dict (input_dict)
+ return xml_record.toxml()
+
+ def dump(self, dump_parents=False):
+ for key in self.fields:
+ if key == 'gid' and self.gid:
+ gid = GID(string=self.gid)
+ print " %s:" % key
+ gid.dump(8, dump_parents)
+ elif getattr(self,key,None):
+ print " %s: %s" % (key, getattr(self,key))
+
+# # only intended for debugging
+# def inspect (self, logger, message=""):
+# logger.info("%s -- Inspecting AlchemyObj -- attrs"%message)
+# for k in dir(self):
+# if not k.startswith('_'):
+# logger.info (" %s: %s"%(k,getattr(self,k)))
+# logger.info("%s -- Inspecting AlchemyObj -- __dict__"%message)
+# d=self.__dict__
+# for (k,v) in d.iteritems():
+# logger.info("[%s]=%s"%(k,v))
+
+
+##############################
+# various kinds of records are implemented as an inheritance hierarchy
+# RegRecord is the base class for all actual variants
+# a first draft was using 'type' as the discriminator for the inheritance
+# but we had to define another more internal column (classtype) so we
+# accomodate variants in types like authority+am and the like
+
+class RegRecord (Base,AlchemyObj):
+ # xxx tmp would be 'records'
+ __tablename__ = 'records'
+ record_id = Column (Integer, primary_key=True)
+ # this is the discriminator that tells which class to use
+ classtype = Column (String)
+ type = Column (String)
+ hrn = Column (String)
+ gid = Column (String)
+ authority = Column (String)
+ peer_authority = Column (String)
+ pointer = Column (Integer, default=-1)
+ date_created = Column (DateTime)
+ last_updated = Column (DateTime)
+ # use the 'type' column to decide which subclass the object is of
+ __mapper_args__ = { 'polymorphic_on' : classtype }
+
+ fields = [ 'type', 'hrn', 'gid', 'authority', 'peer_authority' ]
+ def __init__ (self, type=None, hrn=None, gid=None, authority=None, peer_authority=None,
+ pointer=None, dict=None):
+ if type: self.type=type
+ if hrn: self.hrn=hrn
+ if gid:
+ if isinstance(gid, StringTypes): self.gid=gid
+ else: self.gid=gid.save_to_string(save_parents=True)
+ if authority: self.authority=authority
+ if peer_authority: self.peer_authority=peer_authority
+ if pointer: self.pointer=pointer
+ if dict: self.load_from_dict (dict)
+
+ def __repr__(self):
+ result="[Record id=%s, type=%s, hrn=%s, authority=%s, pointer=%s" % \
+ (self.record_id, self.type, self.hrn, self.authority, self.pointer)
+ # skip the uniform '--- BEGIN CERTIFICATE --' stuff
+ if self.gid: result+=" gid=%s..."%self.gid[28:36]
+ else: result+=" nogid"
+ result += "]"
+ return result
+
+ @validates ('gid')
+ def validate_gid (self, key, gid):
+ if gid is None: return
+ elif isinstance(gid, StringTypes): return gid
+ else: return gid.save_to_string(save_parents=True)
+
+ # xxx - there might be smarter ways to handle get/set'ing gid using validation hooks
+ def get_gid_object (self):
+ if not self.gid: return None
+ else: return GID(string=self.gid)
+
+ def just_created (self):
+ now=datetime.now()
+ self.date_created=now
+ self.last_updated=now
+
+ def just_updated (self):
+ now=datetime.now()
+ self.last_updated=now
+
+##############################
+class RegUser (RegRecord):
+ __tablename__ = 'users'
+ # these objects will have type='user' in the records table
+ __mapper_args__ = { 'polymorphic_identity' : 'user' }
+ record_id = Column (Integer, ForeignKey ("records.record_id"), primary_key=True)
+ email = Column ('email', String)
+
+ # append stuff at the end of the record __repr__
+ def __repr__ (self):
+ result = RegRecord.__repr__(self).replace("Record","User")
+ result.replace ("]"," email=%s"%self.email)
+ return result
+
+ @validates('email')
+ def validate_email(self, key, address):
+ assert '@' in address
+ return address
+
+class RegAuthority (RegRecord):
+ __tablename__ = 'authorities'
+ __mapper_args__ = { 'polymorphic_identity' : 'authority' }
+ record_id = Column (Integer, ForeignKey ("records.record_id"), primary_key=True)
+
+ # no proper data yet, just hack the typename
+ def __repr__ (self):
+ return RegRecord.__repr__(self).replace("Record","Authority")
+
+class RegSlice (RegRecord):
+ __tablename__ = 'slices'
+ __mapper_args__ = { 'polymorphic_identity' : 'slice' }
+ record_id = Column (Integer, ForeignKey ("records.record_id"), primary_key=True)
+
+ def __repr__ (self):
+ return RegRecord.__repr__(self).replace("Record","Slice")
+
+class RegNode (RegRecord):
+ __tablename__ = 'nodes'
+ __mapper_args__ = { 'polymorphic_identity' : 'node' }
+ record_id = Column (Integer, ForeignKey ("records.record_id"), primary_key=True)
+
+ def __repr__ (self):
+ return RegRecord.__repr__(self).replace("Record","Node")
+
+##############################
+# although the db needs of course to be reachable,
+# the schema management functions are here and not in alchemy
+# because the actual details of the classes need to be known
+def init_tables(dbsession):
+ logger.info("Initializing db schema and builtin types")
+ # the doc states we could retrieve the engine this way
+ # engine=dbsession.get_bind()
+ # however I'm getting this
+ # TypeError: get_bind() takes at least 2 arguments (1 given)
+ # so let's import alchemy - but not from toplevel
+ from sfa.storage.alchemy import engine
+ Base.metadata.create_all(engine)
+
+def drop_tables(dbsession):
+ logger.info("Dropping tables")
+ # same as for init_tables
+ from sfa.storage.alchemy import engine
+ Base.metadata.drop_all(engine)
+
+##############################
+# create a record of the right type from either a dict or an xml string
+def make_record (dict={}, xml=""):
+ if dict: return make_record_dict (dict)
+ elif xml: return make_record_xml (xml)
+ else: raise Exception("make_record has no input")
+
+# convert an incoming record - typically from xmlrpc - into an object
+def make_record_dict (record_dict):
+ assert ('type' in record_dict)
+ type=record_dict['type'].split('+')[0]
+ if type=='authority':
+ result=RegAuthority (dict=record_dict)
+ elif type=='user':
+ result=RegUser (dict=record_dict)
+ elif type=='slice':
+ result=RegSlice (dict=record_dict)
+ elif type=='node':
+ result=RegNode (dict=record_dict)
+ else:
+ result=RegRecord (dict=record_dict)
+ logger.info ("converting dict into Reg* with type=%s"%type)
+ logger.info ("returning=%s"%result)
+ # xxx todo
+ # register non-db attributes in an extensions field
+ return result
+
+def make_record_xml (xml):
+ xml_record = XML(xml)
+ xml_dict = xml_record.todict()
+ logger.info("load from xml, keys=%s"%xml_dict.keys())
+ return make_record_dict (xml_dict)
+++ /dev/null
-##
-# Implements support for SFA records
-#
-# TODO: Use existing PLC database methods? or keep this separate?
-##
-
-from types import StringTypes
-from sfa.trust.gid import GID
-from sfa.storage.parameter import Parameter
-from sfa.util.xrn import get_authority
-from sfa.storage.row import Row
-from sfa.util.xml import XML
-from sfa.util.sfalogging import logger
-
-class SfaRecord(Row):
- """
- The SfaRecord class implements an SFA Record. A SfaRecord is a tuple
- (Hrn, GID, Type, Info).
-
- Hrn specifies the Human Readable Name of the object
- GID is the GID of the object
- Type is user | authority | slice | component
-
- Info is comprised of the following sub-fields
- pointer = a pointer to the record in the PL database
-
- The pointer is interpreted depending on the type of the record. For example,
- if the type=="user", then pointer is assumed to be a person_id that indexes
- into the persons table.
-
- A given HRN may have more than one record, provided that the records are
- of different types.
- """
-
- table_name = 'sfa'
-
- primary_key = 'record_id'
-
- ### the wsdl generator assumes this is named 'fields'
- internal_fields = {
- 'record_id': Parameter(int, 'An id that uniquely identifies this record', ro=True),
- 'pointer': Parameter(int, 'An id that uniquely identifies this record in an external database ')
- }
-
- fields = {
- 'authority': Parameter(str, "The authority for this record"),
- 'peer_authority': Parameter(str, "The peer authority for this record"),
- 'hrn': Parameter(str, "Human readable name of object"),
- 'gid': Parameter(str, "GID of the object"),
- 'type': Parameter(str, "Record type"),
- 'last_updated': Parameter(int, 'Date and time of last update', ro=True),
- 'date_created': Parameter(int, 'Date and time this record was created', ro=True),
- }
- all_fields = dict(fields.items() + internal_fields.items())
- ##
- # Create an SFA Record
- #
- # @param name if !=None, assign the name of the record
- # @param gid if !=None, assign the gid of the record
- # @param type one of user | authority | slice | component
- # @param pointer is a pointer to a PLC record
- # @param dict if !=None, then fill in this record from the dictionary
-
- def __init__(self, hrn=None, gid=None, type=None, pointer=None, authority=None, peer_authority=None, dict=None, string=None):
- self.dirty = True
- self.hrn = None
- self.gid = None
- self.type = None
- self.pointer = None
- self.set_peer_auth(peer_authority)
- self.set_authority(authority)
- if hrn:
- self.set_name(hrn)
- if gid:
- self.set_gid(gid)
- if type:
- self.set_type(type)
- if pointer:
- self.set_pointer(pointer)
- if dict:
- self.load_from_dict(dict)
- if string:
- self.load_from_string(string)
-
-
- def validate_last_updated(self, last_updated):
- return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
-
- def update(self, new_dict):
- if isinstance(new_dict, list):
- new_dict = new_dict[0]
-
- # Convert any boolean strings to real bools
- for key in new_dict:
- if isinstance(new_dict[key], StringTypes):
- if new_dict[key].lower() in ["true"]:
- new_dict[key] = True
- elif new_dict[key].lower() in ["false"]:
- new_dict[key] = False
- dict.update(self, new_dict)
-
- ##
- # Set the name of the record
- #
- # @param hrn is a string containing the HRN
-
- def set_name(self, hrn):
- """
- Set the name of the record
- """
- self.hrn = hrn
- self['hrn'] = hrn
- self.dirty = True
-
- def set_authority(self, authority):
- """
- Set the authority
- """
- if not authority:
- authority = ""
- self.authority = authority
- self['authority'] = authority
- self.dirty = True
-
-
- ##
- # Set the GID of the record
- #
- # @param gid is a GID object or the string representation of a GID object
-
- def set_gid(self, gid):
- """
- Set the GID of the record
- """
-
- if isinstance(gid, StringTypes):
- self.gid = gid
- self['gid'] = gid
- else:
- self.gid = gid.save_to_string(save_parents=True)
- self['gid'] = gid.save_to_string(save_parents=True)
- self.dirty = True
-
- ##
- # Set the type of the record
- #
- # @param type is a string: user | authority | slice | component
-
- def set_type(self, type):
- """
- Set the type of the record
- """
- self.type = type
- self['type'] = type
- self.dirty = True
-
- ##
- # Set the pointer of the record
- #
- # @param pointer is an integer containing the ID of a PLC record
-
- def set_pointer(self, pointer):
- """
- Set the pointer of the record
- """
- self.pointer = pointer
- self['pointer'] = pointer
- self.dirty = True
-
-
- def set_peer_auth(self, peer_authority):
- self.peer_authority = peer_authority
- self['peer_authority'] = peer_authority
- self.dirty = True
-
- ##
- # Return the name (HRN) of the record
-
- def get_name(self):
- """
- Return the name (HRN) of the record
- """
- return self.hrn
-
- ##
- # Return the type of the record
-
- def get_type(self):
- """
- Return the type of the record
- """
- return self.type
-
- ##
- # Return the pointer of the record. The pointer is an integer that may be
- # used to look up the record in the PLC database. The evaluation of pointer
- # depends on the type of the record
-
- def get_pointer(self):
- """
- Return the pointer of the record. The pointer is an integer that may be
- used to look up the record in the PLC database. The evaluation of pointer
- depends on the type of the record
- """
- return self.pointer
-
- ##
- # Return the GID of the record, in the form of a GID object
- # TODO: not the best name for the function, because we have things called
- # gidObjects in the Cred
-
- def get_gid_object(self):
- """
- Return the GID of the record, in the form of a GID object
- """
- return GID(string=self.gid)
-
- ##
- # Returns the value of a field
-
- def get_field(self, fieldname, default=None):
- # sometimes records act like classes, and sometimes they act like dicts
- try:
- return getattr(self, fieldname)
- except AttributeError:
- try:
- return self[fieldname]
- except KeyError:
- if default != None:
- return default
- else:
- raise
-
- ##
- # Returns a list of field names in this record.
-
- def get_field_names(self):
- """
- Returns a list of field names in this record.
- """
- return self.fields.keys()
-
- ##
- # Given a field name ("hrn", "gid", ...) return the value of that field.
- #
- # @param fieldname is the name of field to be returned
-
- def get_field_value_string(self, fieldname):
- """
- Given a field name ("hrn", "gid", ...) return the value of that field.
- """
- if fieldname == "authority":
- val = get_authority(self['hrn'])
- else:
- try:
- val = getattr(self, fieldname)
- except:
- val = self[fieldname]
- if isinstance(val, str):
- return "'" + str(val) + "'"
- else:
- return str(val)
-
- ##
- # Given a list of field names, return a list of values for those public.
- #
- # @param fieldnames is a list of field names
-
- def get_field_value_strings(self, fieldnames):
- """
- Given a list of field names, return a list of values for those public.
- """
- return [ self.get_field_value_string (fieldname) for fieldname in fieldnames ]
-
- ##
- # Return the record in the form of a dictionary
-
- def as_dict(self):
- """
- Return the record in the form of a dictionary
- """
- return dict(self)
-
- ##
- # Load the record from a dictionary
- #
- # @param dict dictionary to load record public from
-
- def load_from_dict(self, dict):
- """
- Load the record from a dictionary
- """
-
- self.set_name(dict['hrn'])
- gidstr = dict.get("gid", None)
- if gidstr:
- self.set_gid(dict['gid'])
-
- if "pointer" in dict:
- self.set_pointer(dict['pointer'])
-
- self.set_type(dict['type'])
- self.update(dict)
-
- ##
- # Save the record to a string. The string contains an XML representation of
- # the record.
-
- def save_to_string(self):
- """
- Save the record to a string. The string contains an XML representation of
- the record.
- """
- recorddict = self.as_dict()
- filteredDict = dict([(key, val) for (key, val) in recorddict.iteritems() if key in self.fields.keys()])
- record = XML('<record/>')
- record.parse_dict(filteredDict)
- str = record.toxml()
- return str
-
- ##
- # Load the record from a string. The string is assumed to contain an XML
- # representation of the record.
-
- def load_from_string(self, str):
- """
- Load the record from a string. The string is assumed to contain an XML
- representation of the record.
- """
- #dict = xmlrpclib.loads(str)[0][0]
-
- record = XML(str)
- self.load_from_dict(record.todict())
-
- ##
- # Dump the record to stdout
- #
- # @param dump_parents if true, then the parents of the GID will be dumped
-
- def dump(self, dump_parents=False):
- """
- Walk tree and dump records.
- """
- #print "RECORD", self.name
- #print " hrn:", self.name
- #print " type:", self.type
- #print " gid:"
- #if (not self.gid):
- # print " None"
- #else:
- # self.get_gid_object().dump(8, dump_parents)
- #print " pointer:", self.pointer
-
- order = SfaRecord.fields.keys()
- for key in self.keys():
- if key not in order:
- order.append(key)
- for key in order:
- if key in self and key in self.fields:
- if key in 'gid' and self[key]:
- gid = GID(string=self[key])
- print " %s:" % key
- gid.dump(8, dump_parents)
- else:
- print " %s: %s" % (key, self[key])
-
- def summary_string(self):
- return "Record(record_id=%s, hrn=%s, type=%s, authority=%s, pointer=%s)" % \
- (self.get('record_id'), self.get('hrn'), self.get('type'), self.get('authority'), \
- self.get('pointer'))
-
- def getdict(self):
- return dict(self)
-
- def sync(self):
- """
- Sync this record with the database.
- """
- from sfa.storage.table import SfaTable
- table = SfaTable()
- filter = {}
- if self.get('record_id'):
- filter['record_id'] = self.get('record_id')
- if self.get('hrn') and self.get('type'):
- filter['hrn'] = self.get('hrn')
- filter['type'] = self.get('type')
- if self.get('pointer'):
- filter['pointer'] = self.get('pointer')
- existing_records = table.find(filter)
- if not existing_records:
- table.insert(self)
- else:
- existing_record = existing_records[0]
- self['record_id'] = existing_record['record_id']
- table.update(self)
-
- def delete(self):
- """
- Remove record from the database.
- """
- from sfa.storage.table import SfaTable
- table = SfaTable()
- if self.get('record_id'):
- filter['record_id'] = self.get('record_id')
- if self.get('hrn') and self.get('type'):
- filter['hrn'] = self.get('hrn')
- filter['type'] = self.get('type')
- if self.get('pointer'):
- filter['pointer'] = self.get('pointer')
- existing_records = table.find(filter)
- for record in existing_records:
- table.remove(record)
-
-class UserRecord(SfaRecord):
-
- fields = {
- 'email': Parameter(str, 'email'),
- 'first_name': Parameter(str, 'First name'),
- 'last_name': Parameter(str, 'Last name'),
- 'phone': Parameter(str, 'Phone Number'),
- 'keys': Parameter(str, 'Public key'),
- 'slices': Parameter([str], 'List of slices this user belongs to'),
- }
- fields.update(SfaRecord.fields)
-
-class SliceRecord(SfaRecord):
- fields = {
- 'name': Parameter(str, 'Slice name'),
- 'url': Parameter(str, 'Slice url'),
- 'expires': Parameter(int, 'Date and time this slice exipres'),
- 'researcher': Parameter([str], 'List of users for this slice'),
- 'PI': Parameter([str], 'List of PIs responsible for this slice'),
- 'description': Parameter([str], 'Description of this slice'),
- }
- fields.update(SfaRecord.fields)
-
-
-class NodeRecord(SfaRecord):
- fields = {
- 'hostname': Parameter(str, 'This nodes dns name'),
- 'node_type': Parameter(str, 'Type of node this is'),
- 'node_type': Parameter(str, 'Type of node this is'),
- 'latitude': Parameter(str, 'latitude'),
- 'longitude': Parameter(str, 'longitude'),
- }
- fields.update(SfaRecord.fields)
-
-
-class AuthorityRecord(SfaRecord):
- fields = {
- 'name': Parameter(str, 'Name'),
- 'login_base': Parameter(str, 'login base'),
- 'enabled': Parameter(bool, 'Is this site enabled'),
- 'url': Parameter(str, 'URL'),
- 'nodes': Parameter([str], 'List of nodes at this site'),
- 'operator': Parameter([str], 'List of operators'),
- 'researcher': Parameter([str], 'List of researchers'),
- 'PI': Parameter([str], 'List of Principal Investigators'),
- }
- fields.update(SfaRecord.fields)
-
-
+++ /dev/null
---
--- SFA database schema
---
-
-SET client_encoding = 'UNICODE';
-
---------------------------------------------------------------------------------
--- Version
---------------------------------------------------------------------------------
-
--- Database version
-CREATE TABLE sfa_db_version (
- version integer NOT NULL,
- subversion integer NOT NULL DEFAULT 0
-) WITH OIDS;
-
--- for upgrades/migrations
-
-INSERT INTO sfa_db_version (version, subversion) VALUES (1, 1);
-
---------------------------------------------------------------------------------
--- Aggregates and store procedures
---------------------------------------------------------------------------------
-
--- Like MySQL GROUP_CONCAT(), this function aggregates values into a
--- PostgreSQL array.
-CREATE AGGREGATE array_accum (
- sfunc = array_append,
- basetype = anyelement,
- stype = anyarray,
- initcond = '{}'
-);
-
--- Valid record types
-CREATE TABLE record_types (
- record_type text PRIMARY KEY
-) WITH OIDS;
-INSERT INTO record_types (record_type) VALUES ('authority');
-INSERT INTO record_types (record_type) VALUES ('authority+sa');
-INSERT INTO record_types (record_type) VALUES ('authority+am');
-INSERT INTO record_types (record_type) VALUES ('authority+sm');
-INSERT INTO record_types (record_type) VALUES ('user');
-INSERT INTO record_types (record_type) VALUES ('slice');
-INSERT INTO record_types (record_type) VALUES ('node');
-
-
--- main table
-CREATE TABLE records (
- record_id serial PRIMARY KEY ,
- hrn text NOT NULL,
- authority text NOT NULL,
- peer_authority text,
- gid text,
- type text REFERENCES record_types,
- pointer integer,
- date_created timestamp without time zone NOT NULL DEFAULT CURRENT_TIMESTAMP,
- last_updated timestamp without time zone NOT NULL DEFAULT CURRENT_TIMESTAMP
-);
-CREATE INDEX sfa_hrn_ids on records (hrn);
-CREATE INDEX sfa_type_ids on records (type);
-CREATE INDEX sfa_authority_ids on records (authority);
-CREATE INDEX sfa_peer_authority_ids on records (peer_authority);
-CREATE INDEX sfa_pointer_ids on records (pointer);
+++ /dev/null
-#
-# implements support for SFA records stored in db tables
-#
-# TODO: Use existing PLC database methods? or keep this separate?
-
-from types import StringTypes
-
-from sfa.util.config import Config
-
-from sfa.storage.parameter import Parameter
-from sfa.storage.filter import Filter
-from sfa.storage.PostgreSQL import PostgreSQL
-from sfa.storage.record import SfaRecord, AuthorityRecord, NodeRecord, SliceRecord, UserRecord
-
-class SfaTable(list):
-
- SFA_TABLE_PREFIX = "records"
-
- def __init__(self, record_filter = None):
-
- # pgsql doesn't like table names with "." in them, to replace it with "$"
- self.tablename = SfaTable.SFA_TABLE_PREFIX
- self.config = Config()
- self.db = PostgreSQL(self.config)
-
- if record_filter:
- records = self.find(record_filter)
- for record in records:
- self.append(record)
-
- def db_fields(self, obj=None):
-
- db_fields = self.db.fields(self.SFA_TABLE_PREFIX)
- return dict( [ (key,value) for (key, value) in obj.iteritems() \
- if key in db_fields and
- self.is_writable(key, value, SfaRecord.fields)] )
-
- @staticmethod
- def is_writable (key,value,dict):
- # if not mentioned, assume it's writable (e.g. deleted ...)
- if key not in dict: return True
- # if mentioned but not linked to a Parameter object, idem
- if not isinstance(dict[key], Parameter): return True
- # if not marked ro, it's writable
- if not dict[key].ro: return True
-
- return False
-
-
- def clear (self):
- self.db.do("DELETE from %s"%self.tablename)
- self.db.commit()
-
- # what sfa-nuke does
- def nuke (self):
- self.clear()
-
- def remove(self, record):
- params = {'record_id': record['record_id']}
- template = "DELETE FROM %s " % self.tablename
- sql = template + "WHERE record_id = %(record_id)s"
- self.db.do(sql, params)
-
- # if this is a site, remove all records where 'authority' == the
- # site's hrn
- if record['type'] == 'authority':
- params = {'authority': record['hrn']}
- sql = template + "WHERE authority = %(authority)s"
- self.db.do(sql, params)
- self.db.commit()
-
- def insert(self, record):
- db_fields = self.db_fields(record)
- keys = db_fields.keys()
- values = [self.db.param(key, value) for (key, value) in db_fields.iteritems()]
- query_str = "INSERT INTO " + self.tablename + \
- "(" + ",".join(keys) + ") " + \
- "VALUES(" + ",".join(values) + ")"
- self.db.do(query_str, db_fields)
- self.db.commit()
- result = self.find({'hrn': record['hrn'], 'type': record['type'], 'peer_authority': record['peer_authority']})
- if not result:
- record_id = None
- elif isinstance(result, list):
- record_id = result[0]['record_id']
- else:
- record_id = result['record_id']
-
- return record_id
-
- def update(self, record):
- db_fields = self.db_fields(record)
- keys = db_fields.keys()
- values = [self.db.param(key, value) for (key, value) in db_fields.iteritems()]
- columns = ["%s = %s" % (key, value) for (key, value) in zip(keys, values)]
- query_str = "UPDATE %s SET %s WHERE record_id = %s" % \
- (self.tablename, ", ".join(columns), record['record_id'])
- self.db.do(query_str, db_fields)
- self.db.commit()
-
- def quote_string(self, value):
- return str(self.db.quote(value))
-
- def quote(self, value):
- return self.db.quote(value)
-
- def find(self, record_filter = None, columns=None):
- if not columns:
- columns = "*"
- else:
- columns = ",".join(columns)
- sql = "SELECT %s FROM %s WHERE True " % (columns, self.tablename)
-
- if isinstance(record_filter, (list, tuple, set)):
- ints = filter(lambda x: isinstance(x, (int, long)), record_filter)
- strs = filter(lambda x: isinstance(x, StringTypes), record_filter)
- record_filter = Filter(SfaRecord.all_fields, {'record_id': ints, 'hrn': strs})
- sql += "AND (%s) %s " % record_filter.sql("OR")
- elif isinstance(record_filter, dict):
- record_filter = Filter(SfaRecord.all_fields, record_filter)
- sql += " AND (%s) %s" % record_filter.sql("AND")
- elif isinstance(record_filter, StringTypes):
- record_filter = Filter(SfaRecord.all_fields, {'hrn':[record_filter]})
- sql += " AND (%s) %s" % record_filter.sql("AND")
- elif isinstance(record_filter, int):
- record_filter = Filter(SfaRecord.all_fields, {'record_id':[record_filter]})
- sql += " AND (%s) %s" % record_filter.sql("AND")
-
- results = self.db.selectall(sql)
- if isinstance(results, dict):
- results = [results]
- return results
-
- def findObjects(self, record_filter = None, columns=None):
-
- results = self.find(record_filter, columns)
- result_rec_list = []
- for result in results:
- if result['type'] in ['authority']:
- result_rec_list.append(AuthorityRecord(dict=result))
- elif result['type'] in ['node']:
- result_rec_list.append(NodeRecord(dict=result))
- elif result['type'] in ['slice']:
- result_rec_list.append(SliceRecord(dict=result))
- elif result['type'] in ['user']:
- result_rec_list.append(UserRecord(dict=result))
- else:
- result_rec_list.append(SfaRecord(dict=result))
- return result_rec_list
-
-
from sfa.util.xrn import urn_to_hrn, hrn_authfor_hrn
# 2 weeks, in seconds
-DEFAULT_CREDENTIAL_LIFETIME = 86400 * 14
+DEFAULT_CREDENTIAL_LIFETIME = 86400 * 31
# TODO:
self.decode()
return self.gidObject.get_printable_subject()
+ # sounds like this should be __repr__ instead ??
def get_summary_tostring(self):
if not self.gidObject:
self.decode()
rl.add("bind")
rl.add("control")
rl.add("info")
+# wouldn't that be authority+cm instead ?
elif type == "component":
rl.add("operator")
return rl
except:
- from sfa.util.logging import logger
+ from sfa.util.sfalogging import logger
def run_sfatables (_,__,___, rspec, ____=None):
logger.warning("Cannot import sfatables.runtime, please install package sfa-sfatables")
return rspec
For safety this can also handle inputs that are either timestamps, or datetimes
"""
-
+ # perpare the input for the checks below by
+ # casting strings ('1327098335') to ints
+ if isinstance(input, StringTypes):
+ try:
+ input = int(input)
+ except ValueError:
+ pass
+
if isinstance (input, datetime.datetime):
logger.warn ("argument to utcparse already a datetime - doing nothing")
return input
return time.gmtime(datetime_to_epoch(input))
def datetime_to_epoch(input):
- return time.mktime(input.timetuple())
+ return int(time.mktime(input.timetuple()))
# set schema
for key in self.root.attrib.keys():
if key.endswith('schemaLocation'):
- # schema location should be at the end of the list
- schema_parts = self.root.attrib[key].split(' ')
+ # schemaLocation should be at the end of the list.
+ # Use list comprehension to filter out empty strings
+ schema_parts = [x for x in self.root.attrib[key].split(' ') if x]
self.schema = schema_parts[1]
namespace, schema = schema_parts[0], schema_parts[1]
break
# if not type:
# debug_logger.debug("type-less Xrn's are not safe")
+ def __repr__ (self):
+ result="<XRN u=%s h=%s"%(self.urn,self.hrn)
+ if hasattr(self,'leaf'): result += " leaf=%s"%self.leaf
+ if hasattr(self,'authority'): result += " auth=%s"%self.authority
+ result += ">"
+ return result
+
def get_urn(self): return self.urn
def get_hrn(self): return self.hrn
def get_type(self): return self.type
self._normalize()
return self.leaf
- def get_authority_hrn(self):
+ def get_authority_hrn(self):
self._normalize()
return '.'.join( self.authority )
from testKeypair import *
# xxx broken-test
#from testHierarchy import *
-from testRecord import *
+from testStorage import *
if __name__ == "__main__":
unittest.main()
import unittest
from sfa.trust.gid import *
from sfa.util.config import *
-from sfa.storage.record import *
+from sfa.storage.persistentobjs import RegRecord
-class TestRecord(unittest.TestCase):
+class TestStorage(unittest.TestCase):
def setUp(self):
pass
def testCreate(self):
- r = SfaRecord()
+ r = RegRecord(type='authority',hrn='foo.bar')
if __name__ == "__main__":
unittest.main()
git://git.onelab.eu / sfa.git / commitdiff