From: Sandrine Avakian Date: Thu, 15 Dec 2011 15:30:45 +0000 (+0100) Subject: Merge branch 'master' into senslab2 X-Git-Tag: sfa-2.1-24~3^2~255 X-Git-Url: http://git.onelab.eu/?p=sfa.git;a=commitdiff_plain;h=02a4a59a4a49183ec8c631cc03243b97f4602d50;hp=03dc21a0ac95b4fbe8c950c8dc052955d281cc44 Merge branch 'master' into senslab2 Conflicts: sfa/client/sfi.py sfa/methods/CreateSliver.py sfa/plc/plslices.py --- diff --git a/Makefile b/Makefile index 6d770b4e..4228020f 100644 --- a/Makefile +++ b/Makefile @@ -30,10 +30,15 @@ sfa/util/version.py: sfa/util/version.py.in xmlbuilder-install: cd xmlbuilder-0.9 && python setup.py install --root=$(DESTDIR) && cd - - + rm -rf $(DESTDIR)/usr/lib*/python*/site-packages/*egg-info + +# postinstall steps - various cleanups and tweaks for a nicer rpm python-install: python setup.py install --root=$(DESTDIR) chmod 444 $(DESTDIR)/etc/sfa/default_config.xml + rm -rf $(DESTDIR)/usr/lib*/python*/site-packages/*egg-info + rm -rf $(DESTDIR)/usr/lib*/python*/site-packages/sfa/storage/sfa.sql + (cd $(DESTDIR)/usr/bin ; ln -s sfi.py sfi; ln -s sfascan.py sfascan) python-clean: version-clean python setup.py clean @@ -89,9 +94,13 @@ sfa/methods/__init__.py: force: ########## +# a lot of stuff in the working dir is just noise +scan: + @find . -type f | egrep -v '^\./\.|/\.git/|/\.svn/|TAGS|AA-|~$$|egg-info|\.(py[co]|doc|html|pdf|png|svg|out|bak|dg)$$' tags: - find . -type f | egrep -v '/\.git/|/\.svn/|TAGS|AA-|~$$|\.(py[co]|doc|html|pdf|png|svg|out|bak|xml|dg)$$' | xargs etags -.PHONY: tags + $(MAKE) scan | xargs etags + +.PHONY: scan tags signatures: (cd sfa/methods; grep 'def.*call' *.py > SIGNATURES) @@ -123,13 +132,11 @@ RSYNC_EXCLUDES := --exclude .svn --exclude .git --exclude '*~' --exclude TAGS $ RSYNC_COND_DRY_RUN := $(if $(findstring n,$(MAKEFLAGS)),--dry-run,) RSYNC := rsync -a -v $(RSYNC_COND_DRY_RUN) --no-owner $(RSYNC_EXCLUDES) -CLIENTS = sfi.py getNodes.py getRecord.py setRecord.py \ -sfiAddAttribute.py sfiAddSliver.py sfiDeleteAttribute.py sfiDeleteSliver.py sfiListNodes.py \ -sfiListSlivers.py sfadump.py +CLIENTS = $(shell ls sfa/clientbin/*.py) BINS = ./config/sfa-config-tty ./config/gen-sfa-cm-config.py \ ./sfa/importer/sfa-import-plc.py ./sfa/importer/sfa-nuke-plc.py ./sfa/server/sfa-start.py \ - $(foreach client,$(CLIENTS),./sfa/client/$(client)) + $(CLIENTS) sync: ifeq (,$(SSHURL)) @@ -143,6 +150,7 @@ else +$(RSYNC) $(BINS) $(SSHURL)/usr/bin/ +$(RSYNC) ./init.d/sfa $(SSHURL)/etc/init.d/ +$(RSYNC) ./config/default_config.xml $(SSHURL)/etc/sfa/ + +$(RSYNC) ./sfa/storage/sfa.sql $(SSHURL)/usr/share/sfa/ $(SSHCOMMAND) exec service sfa restart endif @@ -151,5 +159,19 @@ fastsync: +$(RSYNC) ./sfa/ $(SSHURL)/usr/lib\*/python2.\*/site-packages/sfa/ $(SSHCOMMAND) exec service sfa restart -.PHONY: sync +clientsync: + +$(RSYNC) $(BINS) $(SSHURL)/usr/bin/ + +.PHONY: sync fastsync clientsync + ########## +CLIENTLIBFILES= \ +sfa/examples/miniclient.py \ +sfa/__init__.py \ +sfa/client/{sfaserverproxy,sfaclientlib,__init__}.py \ +sfa/trust/{certificate,__init__}.py \ +sfa/util/{sfalogging,faults,genicode,enumeration,__init__}.py + +clientlibsync: + @[ -d "$(CLIENTLIBTARGET)" ] || { echo "You need to set the make variable CLIENTLIBTARGET"; exit 1; } + rsync -av --relative $(CLIENTLIBFILES) $(CLIENTLIBTARGET) diff --git a/config/default_config.xml b/config/default_config.xml index 99c5d374..9156cb32 100644 --- a/config/default_config.xml +++ b/config/default_config.xml @@ -33,12 +33,6 @@ Thierry Parmentelat The human readable name for this interface. - - Aggregate Manager API Version - 1 - The Aggregate API version - - Credential Schema /etc/sfa/credential.xsd @@ -134,6 +128,14 @@ Thierry Parmentelat 12347 The port where the slice manager is to be found. + + + Cache advertisement rspec + false + Enable caching of the global advertisement, as + returned by ListResources without a slice argument. + + @@ -151,12 +153,6 @@ Thierry Parmentelat aggregate manager. - - RSpec Schema - /etc/sfa/pl.rng - The path to the default schema - - Hostname localhost @@ -169,6 +165,14 @@ Thierry Parmentelat 12346 The port where the aggregate is to be found. + + + Cache advertisement rspec + true + Enable caching of the global advertisement, as + returned by ListResources without a slice argument. + + diff --git a/config/sfi_config b/config/sfi_config index 3314790a..60f91a4f 100644 --- a/config/sfi_config +++ b/config/sfi_config @@ -1,4 +1,4 @@ -# user-level configuration for sfi +# user-level configuration example for sfi ### hrn of your authority # For instance, the hrn of a user at the Princeton site on PlanetLab (PLC) @@ -8,17 +8,12 @@ SFI_AUTH='plc.princeton' ### your user hrn # This is your authority. -SFI_USER='plc.princeton.faiyaza' +SFI_USER='plc.princeton.tmack' # ### where to find the registry SFI_REGISTRY='http://www.planet-lab.org:12345/' -# where to find the slice manager +# where to find the slice API +# you can use -s to point to an aggregate URL instead SFI_SM='http://www.planet-lab.org:12347/' - -# where to find the geni aggregate manager -# XX this should be handled by the slice manger -# XX but we cant support it until the geni_am and sfa -# XX interfaces has been unified -SFI_GENI_AM='http://www.planet-lab.org:12348' diff --git a/cron.d/sfa.cron b/cron.d/sfa.cron index c92afb6d..00d27b6c 100644 --- a/cron.d/sfa.cron +++ b/cron.d/sfa.cron @@ -1,8 +1,14 @@ +# +# this example file should be installed in /etc/cron.d owned by root +# SHELL=/bin/bash PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin +# xxx replace this MAILTO=server-msgs@planet-lab.org HOME=/ # # minute hour day-of-month month day-of-week user command -0 * * * * /usr/bin/sfa-import-plc.py > /dev/null 2>&1 -0 0 * * * /usr/bin/sfa-clean-peer-records.py > /dev/null 2>&1 +# once or twice an hour makes sense +0 * * * * root /usr/bin/sfa-import-plc.py >> /var/log/sfa_import.log 2>&1 +# this is needed only if you run RefreshPeer +#0 0 * * * root /usr/bin/sfa-clean-peer-records.py >> /var/log/sfa_import.log 2>&1 diff --git a/setup.py b/setup.py index 724af5d0..a8740fca 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,8 @@ from glob import glob import shutil from distutils.core import setup -bins = [ +scripts = glob("sfa/clientbin/*.py") + \ + [ 'config/sfa-config-tty', 'config/gen-sfa-cm-config.py', 'sfa/importer/sfa-import-plc.py', @@ -18,23 +19,12 @@ bins = [ 'sfa/server/sfa-start.py', 'sfa/server/sfa-clean-peer-records.py', 'sfa/server/sfa_component_setup.py', - 'sfa/client/sfi.py', - 'sfa/client/getNodes.py', - 'sfa/client/getRecord.py', - 'sfa/client/setRecord.py', - 'sfa/client/sfadump.py', - 'sfa/client/sfiListNodes.py', - 'sfa/client/sfiListSlivers.py', - 'sfa/client/sfiAddSliver.py', - 'sfa/client/sfiDeleteSliver.py', - 'sfa/client/sfiAddAttribute.py', - 'sfa/client/sfiDeleteAttribute.py', 'sfatables/sfatables', 'keyconvert/keyconvert.py', 'flashpolicy/sfa_flashpolicy.py', ] -package_dirs = [ +packages = [ 'sfa', 'sfa/trust', 'sfa/storage', @@ -56,10 +46,9 @@ package_dirs = [ 'sfatables/processors', ] - initscripts = [ 'sfa', 'sfa-cm' ] -data_files = [('/etc/sfa/', [ 'config/aggregates.xml', +data_files = [ ('/etc/sfa/', [ 'config/aggregates.xml', 'config/registries.xml', 'config/default_config.xml', 'config/sfi_config', @@ -72,10 +61,11 @@ data_files = [('/etc/sfa/', [ 'config/aggregates.xml', 'sfa/trust/protogeni-rspec-common.xsd', 'flashpolicy/sfa_flashpolicy_config.xml', ]), - ('/etc/sfatables/matches/', glob('sfatables/matches/*.xml')), - ('/etc/sfatables/targets/', glob('sfatables/targets/*.xml')), - ('/etc/init.d/', [ "init.d/%s"%x for x in initscripts ]), - ('/usr/share/sfa/', [ 'sfa/storage/sfa.sql' ] ), + ('/etc/sfatables/matches/', glob('sfatables/matches/*.xml')), + ('/etc/sfatables/targets/', glob('sfatables/targets/*.xml')), + ('/etc/init.d/', [ "init.d/%s"%x for x in initscripts ]), + ('/usr/share/sfa/', [ 'sfa/storage/sfa.sql' ] ), + ('/usr/share/sfa/examples/', glob('sfa/examples/*' ) + [ 'cron.d/sfa.cron' ] ), ] # add sfatables processors as data_files @@ -92,7 +82,7 @@ if sys.argv[1] in ['uninstall', 'remove', 'delete', 'clean']: site_packages_path = [ os.path.join(p,'sfa') for p in python_path if p.endswith('site-packages')] site_packages_path += [ os.path.join(p,'sfatables') for p in python_path if p.endswith('site-packages')] remove_dirs = ['/etc/sfa/', '/etc/sfatables'] + site_packages_path - remove_bins = [ '/usr/bin/' + os.path.basename(bin) for bin in bins ] + remove_bins = [ '/usr/bin/' + os.path.basename(bin) for bin in scripts ] remove_files = remove_bins + [ "/etc/init.d/%s"%x for x in initscripts ] # remove files @@ -112,9 +102,7 @@ if sys.argv[1] in ['uninstall', 'remove', 'delete', 'clean']: else: # avoid repeating what's in the specfile already setup(name='sfa', - packages = package_dirs, + packages = packages, data_files = data_files, - ext_modules = [], - py_modules = [], - scripts = bins) + scripts = scripts) diff --git a/sfa.spec b/sfa.spec index a938cc0f..3236a6cc 100644 --- a/sfa.spec +++ b/sfa.spec @@ -1,6 +1,6 @@ %define name sfa %define version 2.0 -%define taglevel 1 +%define taglevel 5 %define release %{taglevel}%{?pldistro:.%{pldistro}}%{?date:.%{date}} %global python_sitearch %( python -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)" ) @@ -32,7 +32,7 @@ Requires: python-ZSI # for uuidgen - used in db password generation # on f8 this actually comes with e2fsprogs, go figure Requires: util-linux-ng -# xmlbuilder depends on lxml +# xmlbuilder depends on lxml Requires: python-lxml Requires: python-setuptools Requires: python-dateutil @@ -42,6 +42,7 @@ Requires: postgresql-python Requires: python-psycopg2 Requires: pyOpenSSL >= 0.7 Requires: myplc-config +Requires: python-xmlbuilder # python 2.5 has uuid module added, for python 2.4 we still need it. # we can't really check for if we can load uuid as a python module, @@ -84,6 +85,11 @@ Summary: SFA support for flash clients Group: Applications/System Requires: sfa +%package xmlbuilder +Summary: third-party xmlbuilder tool +Group: Applications/System +Provides: python-xmlbuilder + %package tests Summary: unit tests suite for SFA Group: Applications/System @@ -108,10 +114,14 @@ networks. This is the command line interface to manage sfatables %description cm This package implements the SFA interface which serves as a layer between the existing PlanetLab NodeManager interfaces and the SFA API. - + %description flashpolicy This package provides support for adobe flash client applications. - + +%description xmlbuilder +This package contains the xmlbuilder python library, packaged for +convenience as it is not supported by fedora + %description tests Provides some binary unit tests in /usr/share/sfa/tests @@ -124,9 +134,6 @@ make VERSIONTAG="%{version}-%{taglevel}" SCMURL="%{SCMURL}" %install rm -rf $RPM_BUILD_ROOT make VERSIONTAG="%{version}-%{taglevel}" SCMURL="%{SCMURL}" install DESTDIR="$RPM_BUILD_ROOT" -rm -rf $RPM_BUILD_ROOT/%{python_sitelib}/*egg-info -# this gets duplicated -rm -rf $RPM_BUILD_ROOT/%{python_sitelib}/sfa/storage/sfa.sql %clean rm -rf $RPM_BUILD_ROOT @@ -134,7 +141,6 @@ rm -rf $RPM_BUILD_ROOT %files # sfa and sfatables depend on each other. %{python_sitelib}/sfa -%{python_sitelib}/xmlbuilder /etc/init.d/sfa %{_bindir}/sfa-start.py* %{_bindir}/keyconvert.py* @@ -143,6 +149,7 @@ rm -rf $RPM_BUILD_ROOT %config (noreplace) /etc/sfa/aggregates.xml %config (noreplace) /etc/sfa/registries.xml /usr/share/sfa/sfa.sql +/usr/share/sfa/examples /var/www/html/wsdl/*.wsdl %files plc @@ -162,10 +169,12 @@ rm -rf $RPM_BUILD_ROOT %files client %config (noreplace) /etc/sfa/sfi_config -%{_bindir}/sfi* -%{_bindir}/getNodes.py* -%{_bindir}/getRecord.py* +%{_bindir}/sfi*.py* +%{_bindir}/sfi +%{_bindir}/get*.py* %{_bindir}/setRecord.py* +%{_bindir}/sfascan.py* +%{_bindir}/sfascan %{_bindir}/sfadump.py* %files sfatables @@ -182,6 +191,9 @@ rm -rf $RPM_BUILD_ROOT %{_bindir}/sfa_flashpolicy.py* /etc/sfa/sfa_flashpolicy_config.xml +%files xmlbuilder +%{python_sitelib}/xmlbuilder + %files tests %{_datadir}/sfa/tests @@ -212,6 +224,32 @@ fi [ "$1" -ge "1" ] && service sfa-cm restart || : %changelog +* Wed Dec 14 2011 Thierry Parmentelat - sfa-2.0-5 +- client: sfi -a / -p deprecated (use -s instead) +- client: sfi cleaned up +- client: sfi has backward support for APIv1 aggregates again +- server: only APIv2 is supported and should be rather strict +- server: settings for turning on/off caching in sm or am +- server: plc-dependant code has moved from aggregate to pldriver +- server: driver interface extended accordingly + +* Fri Dec 09 2011 Thierry Parmentelat - sfa-2.0-4 +- screwed up previous tag + +* Fri Dec 09 2011 Thierry Parmentelat - sfa-2.0-3 +- client side revisited with a bootstrap library +- client side has a new source layout +- various (nasty) bug fixes wrt options and call_id + +* Tue Dec 06 2011 Thierry Parmentelat - sfa-2.0-2 +- various fixes in rspecs for sfav1&slice tags +- uses 'geni_rspec_version' and not just 'rspec_version' +- example flavour for the max testbed +- embryo for an sfa client library +- topology.py moved into plc +- sql: table is named records; record_types are enforced +- sql: table creation cleaned up + * Wed Nov 30 2011 Thierry Parmentelat - sfa-2.0-1 - cleaned up all references to SFA_*_TYPE in config - enable cache at the aggregate by default diff --git a/sfa/client/sfaclientlib.py b/sfa/client/sfaclientlib.py new file mode 100644 index 00000000..1253267e --- /dev/null +++ b/sfa/client/sfaclientlib.py @@ -0,0 +1,321 @@ +# Thierry Parmentelat -- INRIA +# +# a minimal library for writing "lightweight" SFA clients +# + +import os,os.path + +import sfa.util.sfalogging +# importing sfa.utils.faults does pull a lot of stuff +# OTOH it's imported from Certificate anyways, so.. +from sfa.util.faults import RecordNotFound + +from sfa.client.sfaserverproxy import SfaServerProxy + +# see optimizing dependencies below +from sfa.trust.certificate import Keypair, Certificate + +########## +# a helper class to implement the bootstrapping of crypto. material +# assuming we are starting from scratch on the client side +# what's needed to complete a full slice creation cycle +# (**) prerequisites: +# (*) a local private key +# (*) the corresp. public key in the registry +# (**) step1: a self-signed certificate +# default filename is .sscert +# (**) step2: a user credential +# obtained at the registry with GetSelfCredential +# using the self-signed certificate as the SSL cert +# default filename is .user.cred +# (**) step3: a registry-provided certificate (i.e. a GID) +# obtained at the registry using Resolve +# using the step2 credential as credential +# default filename is .user.gid +# +# From that point on, the GID is used as the SSL certificate +# and the following can be done +# +# (**) retrieve a slice (or authority) credential +# obtained at the registry with GetCredential +# using the (step2) user-credential as credential +# default filename is ..cred +# (**) retrieve a slice (or authority) GID +# obtained at the registry with Resolve +# using the (step2) user-credential as credential +# default filename is ..cred + + +########## Implementation notes +# +# (*) decorators +# +# this implementation is designed as a guideline for +# porting to other languages +# +# the decision to go for decorators aims at focusing +# on the core of what needs to be done when everything +# works fine, and to take caching and error management +# out of the way +# +# for non-pythonic developers, it should be enough to +# implement the bulk of this code, namely the _produce methods +# and to add caching and error management by whichever means +# is available, including inline +# +# (*) self-signed certificates +# +# still with other languages in mind, we've tried to keep the +# dependencies to the rest of the code as low as possible +# +# however this still relies on the sfa.trust.certificate module +# for the initial generation of a self-signed-certificate that +# is associated to the user's ssh-key +# (for user-friendliness, and for smooth operations with planetlab, +# the usage model is to reuse an existing keypair) +# +# there might be a more portable, i.e. less language-dependant way, to +# implement this step by exec'ing the openssl command a known +# successful attempt at this approach that worked for Java is +# documented below +# http://nam.ece.upatras.gr/fstoolkit/trac/wiki/JavaSFAClient +# +#################### + +class SfaClientException (Exception): pass + +class SfaClientBootstrap: + + # dir is mandatory but defaults to '.' + def __init__ (self, user_hrn, registry_url, dir=None, + verbose=False, timeout=None, logger=None): + self.hrn=user_hrn + self.registry_url=registry_url + if dir is None: dir="." + self.dir=dir + self.verbose=verbose + self.timeout=timeout + # default for the logger is to use the global sfa logger + if logger is None: + logger = sfa.util.sfalogging.logger + self.logger=logger + + ######################################## *_produce methods + ### step1 + # unconditionnally create a self-signed certificate + def self_signed_cert_produce (self,output): + self.assert_private_key() + private_key_filename = self.private_key_filename() + keypair=Keypair(filename=private_key_filename) + self_signed = Certificate (subject = self.hrn) + self_signed.set_pubkey (keypair) + self_signed.set_issuer (keypair, self.hrn) + self_signed.sign () + self_signed.save_to_file (output) + self.logger.debug("SfaClientBootstrap: Created self-signed certificate for %s in %s"%\ + (self.hrn,output)) + return output + + ### step2 + # unconditionnally retrieve my credential (GetSelfCredential) + # we always use the self-signed-cert as the SSL cert + def my_credential_produce (self, output): + self.assert_self_signed_cert() + certificate_filename = self.self_signed_cert_filename() + certificate_string = self.plain_read (certificate_filename) + self.assert_private_key() + registry_proxy = SfaServerProxy (self.registry_url, self.private_key_filename(), + certificate_filename) + credential_string=registry_proxy.GetSelfCredential (certificate_string, self.hrn, "user") + self.plain_write (output, credential_string) + self.logger.debug("SfaClientBootstrap: Wrote result of GetSelfCredential in %s"%output) + return output + + ### step3 + # unconditionnally retrieve my GID - use the general form + def my_gid_produce (self,output): + return self.gid_produce (output, self.hrn, "user") + + ### retrieve any credential (GetCredential) unconditionnal form + # we always use the GID as the SSL cert + def credential_produce (self, output, hrn, type): + self.assert_my_gid() + certificate_filename = self.my_gid_filename() + self.assert_private_key() + registry_proxy = SfaServerProxy (self.registry_url, self.private_key_filename(), + certificate_filename) + self.assert_my_credential() + my_credential_string = self.my_credential_string() + credential_string=registry_proxy.GetCredential (my_credential_string, hrn, type) + self.plain_write (output, credential_string) + self.logger.debug("SfaClientBootstrap: Wrote result of GetCredential in %s"%output) + return output + + def slice_credential_produce (self, output, hrn): + return self.credential_produce (output, hrn, "slice") + + def authority_credential_produce (self, output, hrn): + return self.credential_produce (output, hrn, "authority") + + ### retrieve any gid (Resolve) - unconditionnal form + # use my GID when available as the SSL cert, otherwise the self-signed + def gid_produce (self, output, hrn, type ): + try: + self.assert_my_gid() + certificate_filename = self.my_gid_filename() + except: + self.assert_self_signed_cert() + certificate_filename = self.self_signed_cert_filename() + + self.assert_private_key() + registry_proxy = SfaServerProxy (self.registry_url, self.private_key_filename(), + certificate_filename) + credential_string=self.plain_read (self.my_credential()) + records = registry_proxy.Resolve (hrn, credential_string) + records=[record for record in records if record['type']==type] + if not records: + raise RecordNotFound, "hrn %s (%s) unknown to registry %s"%(hrn,type,self.registry_url) + record=records[0] + self.plain_write (output, record['gid']) + self.logger.debug("SfaClientBootstrap: Wrote GID for %s (%s) in %s"% (hrn,type,output)) + return output + + #################### public interface + + # return my_gid, run all missing steps in the bootstrap sequence + def bootstrap_my_gid (self): + self.self_signed_cert() + self.my_credential() + return self.my_gid() + + # once we've bootstrapped we can use this object to issue any other SFA call + # always use my gid + def server_proxy (self, url): + self.assert_my_gid() + return SfaServerProxy (url, self.private_key_filename(), self.my_gid_filename(), + verbose=self.verbose, timeout=self.timeout) + + # now in some cases the self-signed is enough + def server_proxy_simple (self, url): + self.assert_self_signed_cert() + return SfaServerProxy (url, self.private_key_filename(), self.self_signed_cert_filename(), + verbose=self.verbose, timeout=self.timeout) + + # this method can optionnally be invoked to ensure proper + # installation of the private key that belongs to this user + # installs private_key in working dir with expected name -- preserve mode + # typically user_private_key would be ~/.ssh/id_rsa + # xxx should probably check the 2 files are identical + def init_private_key_if_missing (self, user_private_key): + private_key_filename=self.private_key_filename() + if not os.path.isfile (private_key_filename): + key=self.plain_read(user_private_key) + self.plain_write(private_key_filename, key) + os.chmod(private_key_filename,os.stat(user_private_key).st_mode) + self.logger.debug("SfaClientBootstrap: Copied private key from %s into %s"%\ + (user_private_key,private_key_filename)) + + #################### private details + # stupid stuff + def fullpath (self, file): return os.path.join (self.dir,file) + + # the expected filenames for the various pieces + def private_key_filename (self): + return self.fullpath ("%s.pkey"%self.hrn) + def self_signed_cert_filename (self): + return self.fullpath ("%s.sscert"%self.hrn) + def my_credential_filename (self): + return self.credential_filename (self.hrn, "user") + def credential_filename (self, hrn, type): + return self.fullpath ("%s.%s.cred"%(hrn,type)) + def slice_credential_filename (self, hrn): + return self.credential_filename(hrn,'slice') + def authority_credential_filename (self, hrn): + return self.credential_filename(hrn,'authority') + def my_gid_filename (self): + return self.gid_filename ("user", self.hrn) + def gid_filename (self, hrn, type): + return self.fullpath ("%s.%s.gid"%(hrn,type)) + + +# optimizing dependencies +# originally we used classes GID or Credential or Certificate +# like e.g. +# return Credential(filename=self.my_credential()).save_to_string() +# but in order to make it simpler to other implementations/languages.. + def plain_read (self, filename): + infile=file(filename,"r") + result=infile.read() + infile.close() + return result + + def plain_write (self, filename, contents): + outfile=file(filename,"w") + result=outfile.write(contents) + outfile.close() + + def assert_filename (self, filename, kind): + if not os.path.isfile (filename): + raise IOError,"Missing %s file %s"%(kind,filename) + return True + + def assert_private_key (self): return self.assert_filename (self.private_key_filename(),"private key") + def assert_self_signed_cert (self): return self.assert_filename (self.self_signed_cert_filename(),"self-signed certificate") + def assert_my_credential (self): return self.assert_filename (self.my_credential_filename(),"user's credential") + def assert_my_gid (self): return self.assert_filename (self.my_gid_filename(),"user's GID") + + + # decorator to make up the other methods + def get_or_produce (filename_method, produce_method): + def wrap (f): + def wrapped (self, *args, **kw): + filename=filename_method (self, *args, **kw) + if os.path.isfile ( filename ): return filename + try: + produce_method (self, filename, *args, **kw) + return filename + except IOError: + raise + except: + self.logger.log_exc("Could not produce/retrieve %s"%filename) + raise Exception, "Could not produce/retrieve %s"%filename + return wrapped + return wrap + + @get_or_produce (self_signed_cert_filename, self_signed_cert_produce) + def self_signed_cert (self): pass + + @get_or_produce (my_credential_filename, my_credential_produce) + def my_credential (self): pass + + @get_or_produce (my_gid_filename, my_gid_produce) + def my_gid (self): pass + + @get_or_produce (credential_filename, credential_produce) + def credential (self, hrn, type): pass + + @get_or_produce (slice_credential_filename, slice_credential_produce) + def slice_credential (self, hrn): pass + + @get_or_produce (authority_credential_filename, authority_credential_produce) + def authority_credential (self, hrn): pass + + @get_or_produce (gid_filename, gid_produce) + def gid (self, hrn, type ): pass + + + # get the credentials as strings, for inserting as API arguments + def my_credential_string (self): + self.my_credential() + return self.plain_read(self.my_credential_filename()) + def slice_credential_string (self, hrn): + self.slice_credential(hrn) + return self.plain_read(self.slice_credential_filename(hrn)) + def authority_credential_string (self, hrn): + self.authority_credential(hrn) + return self.plain_read(self.authority_credential_filename(hrn)) + + # for consistency + def private_key (self): + self.assert_private_key() + return self.private_key_filename() diff --git a/sfa/client/sfascan.py b/sfa/client/sfascan.py old mode 100755 new mode 100644 index f252378b..fdfa580d --- a/sfa/client/sfascan.py +++ b/sfa/client/sfascan.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import sys, os.path import pickle import time @@ -7,13 +5,16 @@ import socket import traceback from urlparse import urlparse -import pygraphviz +try: + import pygraphviz +except: + print 'Warning, could not import pygraphviz, test mode only' from optparse import OptionParser from sfa.client.sfi import Sfi from sfa.util.sfalogging import logger, DEBUG -import sfa.client.xmlrpcprotocol as xmlrpcprotocol +from sfa.client.sfaserverproxy import SfaServerProxy def url_hostname_port (url): if url.find("://")<0: @@ -148,11 +149,12 @@ class Interface: client.read_config() key_file = client.get_key_file() cert_file = client.get_cert_file(key_file) + logger.debug("using key %s & cert %s"%(key_file,cert_file)) url=self.url() logger.info('issuing GetVersion at %s'%url) # setting timeout here seems to get the call to fail - even though the response time is fast - #server=xmlrpcprotocol.server_proxy(url, key_file, cert_file, verbose=self.verbose, timeout=options.timeout) - server=xmlrpcprotocol.server_proxy(url, key_file, cert_file, verbose=self.verbose) + #server=SfaServerProxy(url, key_file, cert_file, verbose=self.verbose, timeout=options.timeout) + server=SfaServerProxy(url, key_file, cert_file, verbose=self.verbose) self._version=server.GetVersion() except: logger.log_exc("failed to get version") @@ -213,7 +215,7 @@ class Interface: layout['fillcolor']='gray' return layout -class SfaScan: +class Scanner: # provide the entry points (a list of interfaces) def __init__ (self, left_to_right=False, verbose=False): @@ -286,52 +288,60 @@ class SfaScan: logger.error("MISSED interface with node %s"%node) -default_outfiles=['sfa.png','sfa.svg','sfa.dot'] +class SfaScan: -def main(): - usage="%prog [options] url-entry-point(s)" - parser=OptionParser(usage=usage) - parser.add_option("-o","--output",action='append',dest='outfiles',default=[], - help="output filenames (cumulative) - defaults are %r"%default_outfiles) - parser.add_option("-l","--left-to-right",action="store_true",dest="left_to_right",default=False, - help="instead of top-to-bottom") - parser.add_option("-v", "--verbose", action="count", dest="verbose", default=0, - help="verbose - can be repeated for more verbosity") - parser.add_option("-c", "--clean-cache",action='store_true', - dest='clean_cache',default=False, - help='clean/trash version cache and exit') - parser.add_option("-s","--show-cache",action='store_true', - dest='show_cache',default=False, - help='show/display version cache') - - (options,args)=parser.parse_args() - logger.enable_console() - # apply current verbosity to logger - logger.setLevelFromOptVerbose(options.verbose) - # figure if we need to be verbose for these local classes that only have a bool flag - bool_verbose=logger.getBoolVerboseFromOpt(options.verbose) + default_outfiles=['sfa.png','sfa.svg','sfa.dot'] - if options.show_cache: - VersionCache().show() - sys.exit(0) - if options.clean_cache: - VersionCache().clean() - sys.exit(0) - if not args: - parser.print_help() - sys.exit(1) + def main(self): + usage="%prog [options] url-entry-point(s)" + parser=OptionParser(usage=usage) + parser.add_option("-d", "--dir", dest="sfi_dir", + help="config & working directory - default is " + Sfi.default_sfi_dir(), + metavar="PATH", default=Sfi.default_sfi_dir()) + parser.add_option("-o","--output",action='append',dest='outfiles',default=[], + help="output filenames (cumulative) - defaults are %r"%SfaScan.default_outfiles) + parser.add_option("-l","--left-to-right",action="store_true",dest="left_to_right",default=False, + help="instead of top-to-bottom") + parser.add_option("-v", "--verbose", action="count", dest="verbose", default=0, + help="verbose - can be repeated for more verbosity") + parser.add_option("-c", "--clean-cache",action='store_true', + dest='clean_cache',default=False, + help='clean/trash version cache and exit') + parser.add_option("-s","--show-cache",action='store_true', + dest='show_cache',default=False, + help='show/display version cache') - if not options.outfiles: - options.outfiles=default_outfiles - scanner=SfaScan(left_to_right=options.left_to_right, verbose=bool_verbose) - entries = [ Interface(entry) for entry in args ] - g=scanner.graph(entries) - logger.info("creating layout") - g.layout(prog='dot') - for outfile in options.outfiles: - logger.info("drawing in %s"%outfile) - g.draw(outfile) - logger.info("done") + (options,args)=parser.parse_args() + logger.enable_console() + # apply current verbosity to logger + logger.setLevelFromOptVerbose(options.verbose) + # figure if we need to be verbose for these local classes that only have a bool flag + bool_verbose=logger.getBoolVerboseFromOpt(options.verbose) + + if options.show_cache: + VersionCache().show() + sys.exit(0) + if options.clean_cache: + VersionCache().clean() + sys.exit(0) + if not args: + parser.print_help() + sys.exit(1) + + if not options.outfiles: + options.outfiles=SfaScan.default_outfiles + scanner=Scanner(left_to_right=options.left_to_right, verbose=bool_verbose) + entries = [ Interface(entry) for entry in args ] + try: + g=scanner.graph(entries) + logger.info("creating layout") + g.layout(prog='dot') + for outfile in options.outfiles: + logger.info("drawing in %s"%outfile) + g.draw(outfile) + logger.info("done") + # test mode when pygraphviz is not available + except: + entry=entries[0] + print "GetVersion at %s returned %s"%(entry.url(),entry.get_version()) -if __name__ == '__main__': - main() diff --git a/sfa/client/xmlrpcprotocol.py b/sfa/client/sfaserverproxy.py similarity index 71% rename from sfa/client/xmlrpcprotocol.py rename to sfa/client/sfaserverproxy.py index bd741a42..b348126e 100644 --- a/sfa/client/xmlrpcprotocol.py +++ b/sfa/client/sfaserverproxy.py @@ -3,7 +3,11 @@ import xmlrpclib from httplib import HTTPS, HTTPSConnection -from sfa.util.sfalogging import logger +try: + from sfa.util.sfalogging import logger +except: + import logging + logger=logging.getLogger('sfaserverproxy') ## # ServerException, ExceptionUnmarshaller @@ -47,20 +51,18 @@ class XMLRPCTransport(xmlrpclib.Transport): # host may be a string, or a (host, x509-dict) tuple host, extra_headers, x509 = self.get_host_info(host) if need_HTTPSConnection: - #conn = HTTPSConnection(host, None, key_file=self.key_file, cert_file=self.cert_file, timeout=self.timeout) #**(x509 or {})) - conn = HTTPSConnection(host, None, key_file=self.key_file, cert_file=self.cert_file) #**(x509 or {})) + conn = HTTPSConnection(host, None, key_file=self.key_file, cert_file=self.cert_file) else: - #conn = HTTPS(host, None, key_file=self.key_file, cert_file=self.cert_file, timeout=self.timeout) #**(x509 or {})) - conn = HTTPS(host, None, key_file=self.key_file, cert_file=self.cert_file) #**(x509 or {})) - - if hasattr(conn, 'set_timeout'): - conn.set_timeout(self.timeout) + conn = HTTPS(host, None, key_file=self.key_file, cert_file=self.cert_file) # Some logic to deal with timeouts. It appears that some (or all) versions # of python don't set the timeout after the socket is created. We'll do it # ourselves by forcing the connection to connect, finding the socket, and # calling settimeout() on it. (tested with python 2.6) if self.timeout: + if hasattr(conn, 'set_timeout'): + conn.set_timeout(self.timeout) + if hasattr(conn, "_conn"): # HTTPS is a wrapper around HTTPSConnection real_conn = conn._conn @@ -80,6 +82,7 @@ class XMLRPCTransport(xmlrpclib.Transport): class XMLRPCServerProxy(xmlrpclib.ServerProxy): def __init__(self, url, transport, allow_none=True, verbose=False): # remember url for GetVersion + # xxx not sure this is still needed as SfaServerProxy has this too self.url=url xmlrpclib.ServerProxy.__init__(self, url, transport, allow_none=allow_none, verbose=verbose) @@ -87,7 +90,25 @@ class XMLRPCServerProxy(xmlrpclib.ServerProxy): logger.debug ("xml-rpc %s method:%s"%(self.url,attr)) return xmlrpclib.ServerProxy.__getattr__(self, attr) -def server_proxy(url, key_file, cert_file, timeout=None, verbose=False): - transport = XMLRPCTransport(key_file, cert_file, timeout) - return XMLRPCServerProxy(url, transport, allow_none=True, verbose=verbose) +########## the object on which we can send methods that get sent over xmlrpc +class SfaServerProxy: + + def __init__ (self, url, keyfile, certfile, verbose=False, timeout=None): + self.url=url + self.keyfile=keyfile + self.certfile=certfile + self.verbose=verbose + self.timeout=timeout + # an instance of xmlrpclib.ServerProxy + transport = XMLRPCTransport(keyfile, certfile, timeout) + self.serverproxy = XMLRPCServerProxy(url, transport, allow_none=True, verbose=verbose) + + # this is python magic to return the code to run when + # SfaServerProxy receives a method call + # so essentially we send the same method with identical arguments + # to the server_proxy object + def __getattr__(self, name): + def func(*args, **kwds): + return getattr(self.serverproxy, name)(*args, **kwds) + return func diff --git a/sfa/client/sfi.py b/sfa/client/sfi.py old mode 100755 new mode 100644 index b0fa53ca..89714353 --- a/sfa/client/sfi.py +++ b/sfa/client/sfi.py @@ -1,12 +1,13 @@ -#! /usr/bin/env python - -# sfi -- slice-based facility interface +# +# sfi.py - basic SFA command-line client +# the actual binary in sfa/clientbin essentially runs main() +# this module is used in sfascan +# import sys sys.path.append('.') import os, os.path -import tempfile import socket import datetime import codecs @@ -14,6 +15,7 @@ import pickle from lxml import etree from StringIO import StringIO from optparse import OptionParser +from pprint import PrettyPrinter from sfa.trust.certificate import Keypair, Certificate from sfa.trust.gid import GID @@ -31,12 +33,12 @@ from sfa.storage.record import SfaRecord, UserRecord, SliceRecord, NodeRecord, A from sfa.rspecs.rspec import RSpec from sfa.rspecs.rspec_converter import RSpecConverter from sfa.rspecs.version_manager import VersionManager -from sfa.client.return_value import ReturnValue -import sfa.client.xmlrpcprotocol as xmlrpcprotocol +from sfa.client.sfaclientlib import SfaClientBootstrap +from sfa.client.sfaserverproxy import SfaServerProxy, ServerException from sfa.client.client_helper import pg_users_arg, sfa_users_arg +from sfa.client.return_value import ReturnValue -AGGREGATE_PORT=12346 CM_PORT=12346 # utility methods here @@ -165,6 +167,13 @@ class Sfi: required_options=['verbose', 'debug', 'registry', 'sm', 'auth', 'user'] + @staticmethod + def default_sfi_dir (): + if os.path.isfile("./sfi_config"): + return os.getcwd() + else: + return os.path.expanduser("~/.sfi/") + # dummy to meet Sfi's expectations for its 'options' field # i.e. s/t we can do setattr on class DummyOptions: @@ -174,66 +183,78 @@ class Sfi: if options is None: options=Sfi.DummyOptions() for opt in Sfi.required_options: if not hasattr(options,opt): setattr(options,opt,None) - if not hasattr(options,'sfi_dir'): options.sfi_dir=os.path.expanduser("~/.sfi/") - # xxx oops, this is dangerous, sounds like ww sometimes have discrepency - # would be safer to remove self.sfi_dir altogether - self.sfi_dir = options.sfi_dir + if not hasattr(options,'sfi_dir'): options.sfi_dir=Sfi.default_sfi_dir() self.options = options - self.slicemgr = None - self.registry = None self.user = None self.authority = None - self.hashrequest = False self.logger = sfi_logger self.logger.enable_console() + self.available_names = [ tuple[0] for tuple in Sfi.available ] + self.available_dict = dict (Sfi.available) - def create_cmd_parser(self, command, additional_cmdargs=None): - cmdargs = {"list": "authority", - "show": "name", - "remove": "name", - "add": "record", - "update": "record", - "aggregates": "[name]", - "registries": "[name]", - "create_gid": "[name]", - "get_gid": [], - "get_trusted_certs": "cred", - "slices": "", - "resources": "[name]", - "create": "name rspec", - "get_ticket": "name rspec", - "redeem_ticket": "ticket", - "delete": "name", - "reset": "name", - "start": "name", - "stop": "name", - "delegate": "name", - "status": "name", - "renew": "name", - "shutdown": "name", - "version": "", - } - - if additional_cmdargs: - cmdargs.update(additional_cmdargs) - - if command not in cmdargs: + # tuples command-name expected-args in the order in which they should appear in the help + available = [ + ("version", ""), + ("list", "authority"), + ("show", "name"), + ("add", "record"), + ("update", "record"), + ("remove", "name"), + ("slices", ""), + ("resources", "[slice_hrn]"), + ("create", "slice_hrn rspec"), + ("delete", "slice_hrn"), + ("status", "slice_hrn"), + ("start", "slice_hrn"), + ("stop", "slice_hrn"), + ("reset", "slice_hrn"), + ("renew", "slice_hrn time"), + ("shutdown", "slice_hrn"), + ("get_ticket", "slice_hrn rspec"), + ("redeem_ticket", "ticket"), + ("delegate", "name"), + ("create_gid", "[name]"), + ("get_trusted_certs", "cred"), + ] + + def print_command_help (self, options): + verbose=getattr(options,'verbose') + format3="%18s %-15s %s" + line=80*'-' + if not verbose: + print format3%("command","cmd_args","description") + print line + else: + print line + self.create_parser().print_help() + for command in self.available_names: + args=self.available_dict[command] + method=getattr(self,command,None) + doc="" + if method: doc=getattr(method,'__doc__',"") + if not doc: doc="*** no doc found ***" + doc=doc.strip(" \t\n") + doc=doc.replace("\n","\n"+35*' ') + if verbose: + print line + print format3%(command,args,doc) + if verbose: + self.create_command_parser(command).print_help() + + def create_command_parser(self, command): + if command not in self.available_dict: msg="Invalid command\n" msg+="Commands: " - msg += ','.join(cmdargs.keys()) + msg += ','.join(self.available_names) self.logger.critical(msg) sys.exit(2) - parser = OptionParser(usage="sfi [sfi_options] %s [options] %s" \ - % (command, cmdargs[command])) + parser = OptionParser(usage="sfi [sfi_options] %s [cmd_options] %s" \ + % (command, self.available_dict[command])) # user specifies remote aggregate/sm/component if command in ("resources", "slices", "create", "delete", "start", "stop", "restart", "shutdown", "get_ticket", "renew", "status"): - parser.add_option("-a", "--aggregate", dest="aggregate", - default=None, help="aggregate host") - parser.add_option("-p", "--port", dest="port", - default=AGGREGATE_PORT, help="aggregate port") parser.add_option("-c", "--component", dest="component", default=None, help="component hrn") parser.add_option("-d", "--delegate", dest="delegate", default=None, @@ -288,13 +309,9 @@ class Sfi: help="delegate slice credential", metavar="HRN", default=None) if command in ("version"): - parser.add_option("-a", "--aggregate", dest="aggregate", - default=None, help="aggregate host") - parser.add_option("-p", "--port", dest="port", - default=AGGREGATE_PORT, help="aggregate port") parser.add_option("-R","--registry-version", action="store_true", dest="version_registry", default=False, - help="probe registry version instead of slicemgr") + help="probe registry version instead of sliceapi") parser.add_option("-l","--local", action="store_true", dest="version_local", default=False, help="display version of the local client") @@ -305,16 +322,15 @@ class Sfi: def create_parser(self): # Generate command line parser - parser = OptionParser(usage="sfi [options] command [command_options] [command_args]", - description="Commands: gid,list,show,remove,add,update,nodes,slices,resources,create,delete,start,stop,reset") + parser = OptionParser(usage="sfi [sfi_options] command [cmd_options] [cmd_args]", + description="Commands: %s"%(" ".join(self.available_names))) parser.add_option("-r", "--registry", dest="registry", help="root registry", metavar="URL", default=None) - parser.add_option("-s", "--slicemgr", dest="sm", - help="slice manager", metavar="URL", default=None) - default_sfi_dir = os.path.expanduser("~/.sfi/") + parser.add_option("-s", "--sliceapi", dest="sm", default=None, metavar="URL", + help="slice API - in general a SM URL, but can be used to talk to an aggregate") parser.add_option("-d", "--dir", dest="sfi_dir", - help="config & working directory - default is " + default_sfi_dir, - metavar="PATH", default=default_sfi_dir) + help="config & working directory - default is %default", + metavar="PATH", default=Sfi.default_sfi_dir()) parser.add_option("-u", "--user", dest="user", help="user name", metavar="HRN", default=None) parser.add_option("-a", "--auth", dest="auth", @@ -324,91 +340,221 @@ class Sfi: parser.add_option("-D", "--debug", action="store_true", dest="debug", default=False, help="Debug (xml-rpc) protocol messages") - parser.add_option("-p", "--protocol", dest="protocol", default="xmlrpc", - help="RPC protocol (xmlrpc or soap)") - parser.add_option("-k", "--hashrequest", - action="store_true", dest="hashrequest", default=False, - help="Create a hash of the request that will be authenticated on the server") + # would it make sense to use ~/.ssh/id_rsa as a default here ? + parser.add_option("-k", "--private-key", + action="store", dest="user_private_key", default=None, + help="point to the private key file to use if not yet installed in sfi_dir") parser.add_option("-t", "--timeout", dest="timeout", default=None, - help="Amout of time tom wait before timing out the request") + help="Amout of time to wait before timing out the request") + parser.add_option("-?", "--commands", + action="store_true", dest="command_help", default=False, + help="one page summary on commands & exit") parser.disable_interspersed_args() return parser - def read_config(self): - config_file = os.path.join(self.options.sfi_dir,"sfi_config") - try: - config = Config (config_file) - except: - self.logger.critical("Failed to read configuration file %s"%config_file) - self.logger.info("Make sure to remove the export clauses and to add quotes") - if self.options.verbose==0: - self.logger.info("Re-run with -v for more details") - else: - self.logger.log_exc("Could not read config file %s"%config_file) - sys.exit(1) - - errors = 0 - # Set SliceMgr URL - if (self.options.sm is not None): - self.sm_url = self.options.sm - elif hasattr(config, "SFI_SM"): - self.sm_url = config.SFI_SM - else: - self.logger.error("You need to set e.g. SFI_SM='http://your.slicemanager.url:12347/' in %s" % config_file) - errors += 1 + def print_help (self): + self.sfi_parser.print_help() + self.command_parser.print_help() + + # + # Main: parse arguments and dispatch to command + # + def dispatch(self, command, command_options, command_args): + return getattr(self, command)(command_options, command_args) + + def main(self): + self.sfi_parser = self.create_parser() + (options, args) = self.sfi_parser.parse_args() + if options.command_help: + self.print_command_help(options) + sys.exit(1) + self.options = options + + self.logger.setLevelFromOptVerbose(self.options.verbose) + + if len(args) <= 0: + self.logger.critical("No command given. Use -h for help.") + self.print_command_help(options) + return -1 - # Set Registry URL - if (self.options.registry is not None): - self.reg_url = self.options.registry - elif hasattr(config, "SFI_REGISTRY"): - self.reg_url = config.SFI_REGISTRY - else: - self.logger.errors("You need to set e.g. SFI_REGISTRY='http://your.registry.url:12345/' in %s" % config_file) - errors += 1 - - - # Set user HRN - if (self.options.user is not None): - self.user = self.options.user - elif hasattr(config, "SFI_USER"): - self.user = config.SFI_USER - else: - self.logger.errors("You need to set e.g. SFI_USER='plc.princeton.username' in %s" % config_file) - errors += 1 + command = args[0] + self.command_parser = self.create_command_parser(command) + (command_options, command_args) = self.command_parser.parse_args(args[1:]) + self.command_options = command_options + + self.read_config () + self.bootstrap () + self.logger.info("Command=%s" % command) + + try: + self.dispatch(command, command_options, command_args) + except KeyError: + self.logger.critical ("Unknown command %s"%command) + raise + sys.exit(1) - # Set authority HRN - if (self.options.auth is not None): - self.authority = self.options.auth - elif hasattr(config, "SFI_AUTH"): - self.authority = config.SFI_AUTH - else: - self.logger.error("You need to set e.g. SFI_AUTH='plc.princeton' in %s" % config_file) - errors += 1 + return - if errors: - sys.exit(1) + #################### + def read_config(self): + config_file = os.path.join(self.options.sfi_dir,"sfi_config") + try: + config = Config (config_file) + except: + self.logger.critical("Failed to read configuration file %s"%config_file) + self.logger.info("Make sure to remove the export clauses and to add quotes") + if self.options.verbose==0: + self.logger.info("Re-run with -v for more details") + else: + self.logger.log_exc("Could not read config file %s"%config_file) + sys.exit(1) + + errors = 0 + # Set SliceMgr URL + if (self.options.sm is not None): + self.sm_url = self.options.sm + elif hasattr(config, "SFI_SM"): + self.sm_url = config.SFI_SM + else: + self.logger.error("You need to set e.g. SFI_SM='http://your.slicemanager.url:12347/' in %s" % config_file) + errors += 1 + + # Set Registry URL + if (self.options.registry is not None): + self.reg_url = self.options.registry + elif hasattr(config, "SFI_REGISTRY"): + self.reg_url = config.SFI_REGISTRY + else: + self.logger.errors("You need to set e.g. SFI_REGISTRY='http://your.registry.url:12345/' in %s" % config_file) + errors += 1 + + # Set user HRN + if (self.options.user is not None): + self.user = self.options.user + elif hasattr(config, "SFI_USER"): + self.user = config.SFI_USER + else: + self.logger.errors("You need to set e.g. SFI_USER='plc.princeton.username' in %s" % config_file) + errors += 1 + + # Set authority HRN + if (self.options.auth is not None): + self.authority = self.options.auth + elif hasattr(config, "SFI_AUTH"): + self.authority = config.SFI_AUTH + else: + self.logger.error("You need to set e.g. SFI_AUTH='plc.princeton' in %s" % config_file) + errors += 1 + if errors: + sys.exit(1) # - # Establish Connection to SliceMgr and Registry Servers + # Get various credential and spec files + # + # Establishes limiting conventions + # - conflates MAs and SAs + # - assumes last token in slice name is unique + # + # Bootstraps credentials + # - bootstrap user credential from self-signed certificate + # - bootstrap authority credential from user credential + # - bootstrap slice credential from user credential # - def set_servers(self): - - self.read_config() - # Get key and certificate - key_file = self.get_key_file() - cert_file = self.get_cert_file(key_file) - self.key = Keypair(filename=key_file) - self.key_file = key_file - self.cert_file = cert_file - self.cert = GID(filename=cert_file) - self.logger.info("Contacting Registry at: %s"%self.reg_url) - self.registry = xmlrpcprotocol.server_proxy(self.reg_url, key_file, cert_file, timeout=self.options.timeout, verbose=self.options.debug) - self.logger.info("Contacting Slice Manager at: %s"%self.sm_url) - self.slicemgr = xmlrpcprotocol.server_proxy(self.sm_url, key_file, cert_file, timeout=self.options.timeout, verbose=self.options.debug) - return + + # init self-signed cert, user credentials and gid + def bootstrap (self): + bootstrap = SfaClientBootstrap (self.user, self.reg_url, self.options.sfi_dir) + # if -k is provided, use this to initialize private key + if self.options.user_private_key: + bootstrap.init_private_key_if_missing (self.options.user_private_key) + else: + # trigger legacy compat code if needed + # the name has changed from just .pkey to .pkey + if not os.path.isfile(bootstrap.private_key_filename()): + self.logger.info ("private key not found, trying legacy name") + try: + legacy_private_key = os.path.join (self.options.sfi_dir, "%s.pkey"%get_leaf(self.user)) + self.logger.debug("legacy_private_key=%s"%legacy_private_key) + bootstrap.init_private_key_if_missing (legacy_private_key) + self.logger.info("Copied private key from legacy location %s"%legacy_private_key) + except: + self.logger.log_exc("Can't find private key ") + sys.exit(1) + + # make it bootstrap + bootstrap.bootstrap_my_gid() + # extract what's needed + self.private_key = bootstrap.private_key() + self.my_credential_string = bootstrap.my_credential_string () + self.my_gid = bootstrap.my_gid () + self.bootstrap = bootstrap + + + def my_authority_credential_string(self): + if not self.authority: + self.logger.critical("no authority specified. Use -a or set SF_AUTH") + sys.exit(-1) + return self.bootstrap.authority_credential_string (self.authority) + + def slice_credential_string(self, name): + return self.bootstrap.slice_credential_string (name) + + # xxx should be supported by sfaclientbootstrap as well + def delegate_cred(self, object_cred, hrn, type='authority'): + # the gid and hrn of the object we are delegating + if isinstance(object_cred, str): + object_cred = Credential(string=object_cred) + object_gid = object_cred.get_gid_object() + object_hrn = object_gid.get_hrn() + + if not object_cred.get_privileges().get_all_delegate(): + self.logger.error("Object credential %s does not have delegate bit set"%object_hrn) + return + + # the delegating user's gid + caller_gidfile = self.my_gid() + + # the gid of the user who will be delegated to + delegee_gid = self.bootstrap.gid(hrn,type) + delegee_hrn = delegee_gid.get_hrn() + dcred = object_cred.delegate(delegee_gid, self.private_key, caller_gidfile) + return dcred.save_to_string(save_parents=True) + + # + # Management of the servers + # + + def registry (self): + # cache the result + if not hasattr (self, 'registry_proxy'): + self.logger.info("Contacting Registry at: %s"%self.reg_url) + self.registry_proxy = SfaServerProxy(self.reg_url, self.private_key, self.my_gid, + timeout=self.options.timeout, verbose=self.options.debug) + return self.registry_proxy + + def sliceapi (self): + # cache the result + if not hasattr (self, 'sliceapi_proxy'): + # if the command exposes the --component option, figure it's hostname and connect at CM_PORT + if hasattr(self.command_options,'component') and self.command_options.component: + # resolve the hrn at the registry + node_hrn = self.command_options.component + records = self.registry().Resolve(node_hrn, self.my_credential_string) + records = filter_records('node', records) + if not records: + self.logger.warning("No such component:%r"% opts.component) + record = records[0] + cm_url = "http://%s:%d/"%(record['hostname'],CM_PORT) + self.sliceapi_proxy=SfaServerProxy(cm_url, self.private_key, self.my_gid) + else: + # otherwise use what was provided as --sliceapi, or SFI_SM in the config + self.logger.info("Contacting Slice Manager at: %s"%self.sm_url) + self.sliceapi_proxy = SfaServerProxy(self.sm_url, self.private_key, self.my_gid, + timeout=self.options.timeout, verbose=self.options.debug) + return self.sliceapi_proxy def get_cached_server_version(self, server): # check local cache first @@ -428,194 +574,56 @@ class Sfi: if not version: result = server.GetVersion() version= ReturnValue.get_value(result) - # cache version for 24 hours - cache.add(cache_key, version, ttl= 60*60*24) + # cache version for 20 minutes + cache.add(cache_key, version, ttl= 60*20) self.logger.info("Updating cache file %s" % cache_file) cache.save_to_file(cache_file) return version - + ### resurrect this temporarily so we can support V1 aggregates for a while def server_supports_options_arg(self, server): """ Returns true if server support the optional call_id arg, false otherwise. """ server_version = self.get_cached_server_version(server) + result = False + # xxx need to rewrite this + if int(server_version.get('geni_api')) >= 2: + result = True + return result + + def server_supports_call_id_arg(self, server): + server_version = self.get_cached_server_version(server) + result = False if 'sfa' in server_version and 'code_tag' in server_version: code_tag = server_version['code_tag'] code_tag_parts = code_tag.split("-") - version_parts = code_tag_parts[0].split(".") major, minor = version_parts[0], version_parts[1] rev = code_tag_parts[1] - if int(major) >= 1: - if int(minor) >= 2: - return True - return False - - # - # Get various credential and spec files - # - # Establishes limiting conventions - # - conflates MAs and SAs - # - assumes last token in slice name is unique - # - # Bootstraps credentials - # - bootstrap user credential from self-signed certificate - # - bootstrap authority credential from user credential - # - bootstrap slice credential from user credential - # - - - def get_key_file(self): - file = os.path.join(self.options.sfi_dir, self.user.replace(self.authority + '.', '') + ".pkey") - if (os.path.isfile(file)): - return file - else: - self.logger.error("Key file %s does not exist"%file) - sys.exit(-1) - return - - def get_cert_file(self, key_file): - - cert_file = os.path.join(self.options.sfi_dir, self.user.replace(self.authority + '.', '') + ".cert") - if (os.path.isfile(cert_file)): - # we'd perfer to use Registry issued certs instead of self signed certs. - # if this is a Registry cert (GID) then we are done - gid = GID(filename=cert_file) - if gid.get_urn(): - return cert_file - - # generate self signed certificate - k = Keypair(filename=key_file) - cert = Certificate(subject=self.user) - cert.set_pubkey(k) - cert.set_issuer(k, self.user) - cert.sign() - self.logger.info("Writing self-signed certificate to %s"%cert_file) - cert.save_to_file(cert_file) - self.cert = cert - # try to get registry issued cert - try: - self.logger.info("Getting Registry issued cert") - self.read_config() - # *hack. need to set registyr before _get_gid() is called - self.registry = xmlrpcprotocol.server_proxy(self.reg_url, key_file, cert_file, timeout=self.options.timeout, verbose=self.options.debug) - gid = self._get_gid(type='user') - self.registry = None - self.logger.info("Writing certificate to %s"%cert_file) - gid.save_to_file(cert_file) - except: - self.logger.info("Failed to download Registry issued cert") - - return cert_file - - def get_cached_gid(self, file): - """ - Return a cached gid - """ - gid = None - if (os.path.isfile(file)): - gid = GID(filename=file) - return gid - - # xxx opts unused - def get_gid(self, opts, args): - """ - Get the specify gid and save it to file - """ - hrn = None - if args: - hrn = args[0] - gid = self._get_gid(hrn) - self.logger.debug("Sfi.get_gid-> %s" % gid.save_to_string(save_parents=True)) - return gid - - def _get_gid(self, hrn=None, type=None): - """ - git_gid helper. Retrive the gid from the registry and save it to file. - """ - - if not hrn: - hrn = self.user - - gidfile = os.path.join(self.options.sfi_dir, hrn + ".gid") - gid = self.get_cached_gid(gidfile) - if not gid: - user_cred = self.get_user_cred() - print>>sys.stderr, " \r\n \t SFI.PY _get_gid " - records = self.registry.Resolve(hrn, user_cred.save_to_string(save_parents=True)) - if not records: - raise RecordNotFound(args[0]) - record = records[0] - if type: - record=None - for rec in records: - if type == rec['type']: - record = rec - if not record: - raise RecordNotFound(args[0]) - - gid = GID(string=record['gid']) - self.logger.info("Writing gid to %s"%gidfile) - gid.save_to_file(filename=gidfile) - return gid - - - def get_cached_credential(self, file): - """ - Return a cached credential only if it hasn't expired. - """ - if (os.path.isfile(file)): - credential = Credential(filename=file) - # make sure it isnt expired - if not credential.get_expiration or \ - datetime.datetime.today() < credential.get_expiration(): - return credential - return None - - def get_user_cred(self): - file = os.path.join(self.options.sfi_dir, self.user.replace(self.authority + '.', '') + ".cred") - return self.get_cred(file, 'user', self.user) - - def get_auth_cred(self): - if not self.authority: - self.logger.critical("no authority specified. Use -a or set SF_AUTH") - sys.exit(-1) - file = os.path.join(self.options.sfi_dir, self.authority + ".cred") - return self.get_cred(file, 'authority', self.authority) - - def get_slice_cred(self, name): - file = os.path.join(self.options.sfi_dir, "slice_" + get_leaf(name) + ".cred") - return self.get_cred(file, 'slice', name) - - def get_cred(self, file, type, hrn): - # attempt to load a cached credential - cred = self.get_cached_credential(file) - if not cred: - if type in ['user']: - cert_string = self.cert.save_to_string(save_parents=True) - user_name = self.user.replace(self.authority + ".", '') - if user_name.count(".") > 0: - user_name = user_name.replace(".", '_') - self.user = self.authority + "." + user_name - cred_str = self.registry.GetSelfCredential(cert_string, hrn, "user") - else: - # bootstrap slice credential from user credential - user_cred = self.get_user_cred().save_to_string(save_parents=True) - cred_str = self.registry.GetCredential(user_cred, hrn, type) - - if not cred_str: - self.logger.critical("Failed to get %s credential" % type) - sys.exit(-1) - - cred = Credential(string=cred_str) - cred.save_to_file(file, save_parents=True) - self.logger.info("Writing %s credential to %s" %(type, file)) + if int(major) == 1 and minor == 0 and build >= 22: + result = True + return result + + ### ois = options if supported + # to be used in something like serverproxy.Method (arg1, arg2, *self.ois(api_options)) + def ois (self, server, option_dict): + if self.server_supports_options_arg (server): + return [option_dict] + elif self.server_supports_call_id_arg (server): + return [ unique_call_id () ] + else: + return [] + + ### cis = call_id if supported - like ois + def cis (self, server): + if self.server_supports_call_id_arg (server): + return [ unique_call_id ] + else: + return [] - return cred - - + ######################################## miscell utilities def get_rspec_file(self, rspec): if (os.path.isabs(rspec)): file = rspec @@ -638,115 +646,66 @@ class Sfi: self.logger.critical("No such registry record file %s"%record) sys.exit(1) - def load_publickey_string(self, fn): - f = file(fn, "r") - key_string = f.read() - - # if the filename is a private key file, then extract the public key - if "PRIVATE KEY" in key_string: - outfn = tempfile.mktemp() - cmd = "openssl rsa -in " + fn + " -pubout -outform PEM -out " + outfn - os.system(cmd) - f = file(outfn, "r") - key_string = f.read() - os.remove(outfn) - - return key_string - - # xxx opts undefined - def get_component_proxy_from_hrn(self, hrn): - # direct connection to the nodes component manager interface - user_cred = self.get_user_cred().save_to_string(save_parents=True) - records = self.registry.Resolve(hrn, user_cred) - records = filter_records('node', records) - if not records: - self.logger.warning("No such component:%r"% opts.component) - record = records[0] - - return self.server_proxy(record['hostname'], CM_PORT, self.key_file, self.cert_file) - - def server_proxy(self, host, port, keyfile, certfile): - """ - Return an instance of an xmlrpc server connection - """ - # port is appended onto the domain, before the path. Should look like: - # http://domain:port/path - host_parts = host.split('/') - host_parts[0] = host_parts[0] + ":" + str(port) - url = "http://%s" % "/".join(host_parts) - return xmlrpcprotocol.server_proxy(url, keyfile, certfile, timeout=self.options.timeout, verbose=self.options.debug) - - # xxx opts could be retrieved in self.options - def server_proxy_from_opts(self, opts): - """ - Return instance of an xmlrpc connection to a slice manager, aggregate - or component server depending on the specified opts - """ - server = self.slicemgr - # direct connection to an aggregate - if hasattr(opts, 'aggregate') and opts.aggregate: - server = self.server_proxy(opts.aggregate, opts.port, self.key_file, self.cert_file) - # direct connection to the nodes component manager interface - if hasattr(opts, 'component') and opts.component: - server = self.get_component_proxy_from_hrn(opts.component) - - return server + #========================================================================== # Following functions implement the commands # # Registry-related commands #========================================================================== - def dispatch(self, command, cmd_opts, cmd_args): - return getattr(self, command)(cmd_opts, cmd_args) - - def create_gid(self, opts, args): - if len(args) < 1: - self.print_help() - sys.exit(1) - target_hrn = args[0] - user_cred = self.get_user_cred().save_to_string(save_parents=True) - gid = self.registry.CreateGid(user_cred, target_hrn, self.cert.save_to_string()) - if opts.file: - filename = opts.file + def version(self, options, args): + """ + display an SFA server version (GetVersion) +or version information about sfi itself + """ + if options.version_local: + version=version_core() else: - filename = os.sep.join([self.sfi_dir, '%s.gid' % target_hrn]) - self.logger.info("writing %s gid to %s" % (target_hrn, filename)) - GID(string=gid).save_to_file(filename) - - - # list entires in named authority registry - def list(self, opts, args): + if options.version_registry: + server=self.registry() + else: + server = self.sliceapi() + result = server.GetVersion() + version = ReturnValue.get_value(result) + pprinter = PrettyPrinter(indent=4) + pprinter.pprint(version) + if options.file: + save_variable_to_file(version, options.file, options.fileformat) + + def list(self, options, args): + """ + list entries in named authority registry (List) + """ if len(args)!= 1: self.print_help() sys.exit(1) hrn = args[0] - user_cred = self.get_user_cred().save_to_string(save_parents=True) try: - list = self.registry.List(hrn, user_cred) + list = self.registry().List(hrn, self.my_credential_string) except IndexError: raise Exception, "Not enough parameters for the 'list' command" # filter on person, slice, site, node, etc. # THis really should be in the self.filter_records funct def comment... - list = filter_records(opts.type, list) + list = filter_records(options.type, list) for record in list: print "%s (%s)" % (record['hrn'], record['type']) - if opts.file: - save_records_to_file(opts.file, list, opts.fileformat) + if options.file: + save_records_to_file(options.file, list, options.fileformat) return - # show named registry record - def show(self, opts, args): + def show(self, options, args): + """ + show details about named registry record (Resolve) + """ if len(args)!= 1: self.print_help() sys.exit(1) hrn = args[0] - user_cred = self.get_user_cred().save_to_string(save_parents=True) - records = self.registry.Resolve(hrn, user_cred) - records = filter_records(opts.type, records) + records = self.registry().Resolve(hrn, self.my_credential_string) + records = filter_records(options.type, records) if not records: - print "No record of type", opts.type + self.logger.error("No record of type %s"% options.type) for record in records: if record['type'] in ['user']: record = UserRecord(dict=record) @@ -758,497 +717,478 @@ class Sfi: record = AuthorityRecord(dict=record) else: record = SfaRecord(dict=record) - if (opts.format == "text"): + if (options.format == "text"): record.dump() else: print record.save_to_string() - if opts.file: - save_records_to_file(opts.file, records, opts.fileformat) + if options.file: + save_records_to_file(options.file, records, options.fileformat) return - def delegate(self, opts, args): - - delegee_hrn = args[0] - if opts.delegate_user: - user_cred = self.get_user_cred() - cred = self.delegate_cred(user_cred, delegee_hrn) - elif opts.delegate_slice: - slice_cred = self.get_slice_cred(opts.delegate_slice) - cred = self.delegate_cred(slice_cred, delegee_hrn) - else: - self.logger.warning("Must specify either --user or --slice ") - return - delegated_cred = Credential(string=cred) - object_hrn = delegated_cred.get_gid_object().get_hrn() - if opts.delegate_user: - dest_fn = os.path.join(self.options.sfi_dir, get_leaf(delegee_hrn) + "_" - + get_leaf(object_hrn) + ".cred") - elif opts.delegate_slice: - dest_fn = os.path.join(self.options.sfi_dir, get_leaf(delegee_hrn) + "_slice_" - + get_leaf(object_hrn) + ".cred") - - delegated_cred.save_to_file(dest_fn, save_parents=True) - - self.logger.info("delegated credential for %s to %s and wrote to %s"%(object_hrn, delegee_hrn,dest_fn)) - - def delegate_cred(self, object_cred, hrn): - # the gid and hrn of the object we are delegating - if isinstance(object_cred, str): - object_cred = Credential(string=object_cred) - object_gid = object_cred.get_gid_object() - object_hrn = object_gid.get_hrn() - - if not object_cred.get_privileges().get_all_delegate(): - self.logger.error("Object credential %s does not have delegate bit set"%object_hrn) - return - - # the delegating user's gid - caller_gid = self._get_gid(self.user) - caller_gidfile = os.path.join(self.options.sfi_dir, self.user + ".gid") - - # the gid of the user who will be delegated to - delegee_gid = self._get_gid(hrn) - delegee_hrn = delegee_gid.get_hrn() - delegee_gidfile = os.path.join(self.options.sfi_dir, delegee_hrn + ".gid") - delegee_gid.save_to_file(filename=delegee_gidfile) - dcred = object_cred.delegate(delegee_gidfile, self.get_key_file(), caller_gidfile) - return dcred.save_to_string(save_parents=True) - - # removed named registry record - # - have to first retrieve the record to be removed - def remove(self, opts, args): - auth_cred = self.get_auth_cred().save_to_string(save_parents=True) - if len(args)!=1: - self.print_help() - sys.exit(1) - hrn = args[0] - type = opts.type - if type in ['all']: - type = '*' - return self.registry.Remove(hrn, auth_cred, type) - - # add named registry record - def add(self, opts, args): - auth_cred = self.get_auth_cred().save_to_string(save_parents=True) + def add(self, options, args): + "add record into registry from xml file (Register)" + auth_cred = self.my_authority_credential_string() if len(args)!=1: self.print_help() sys.exit(1) record_filepath = args[0] rec_file = self.get_record_file(record_filepath) record = load_record_from_file(rec_file).as_dict() - return self.registry.Register(record, auth_cred) + return self.registry().Register(record, auth_cred) - # update named registry entry - def update(self, opts, args): - user_cred = self.get_user_cred() + def update(self, options, args): + "update record into registry from xml file (Update)" if len(args)!=1: self.print_help() sys.exit(1) rec_file = self.get_record_file(args[0]) record = load_record_from_file(rec_file) if record['type'] == "user": - if record.get_name() == user_cred.get_gid_object().get_hrn(): - cred = user_cred.save_to_string(save_parents=True) + if record.get_name() == self.user: + cred = self.my_credential_string else: - cred = self.get_auth_cred().save_to_string(save_parents=True) + cred = self.my_authority_credential_string() elif record['type'] in ["slice"]: try: - cred = self.get_slice_cred(record.get_name()).save_to_string(save_parents=True) - except xmlrpcprotocol.ServerException, e: + cred = self.slice_credential_string(record.get_name()) + except ServerException, e: # XXX smbaker -- once we have better error return codes, update this # to do something better than a string compare if "Permission error" in e.args[0]: - cred = self.get_auth_cred().save_to_string(save_parents=True) + cred = self.my_authority_credential_string() else: raise elif record.get_type() in ["authority"]: - cred = self.get_auth_cred().save_to_string(save_parents=True) + cred = self.my_authority_credential_string() elif record.get_type() == 'node': - cred = self.get_auth_cred().save_to_string(save_parents=True) + cred = self.my_authority_credential_string() else: raise "unknown record type" + record.get_type() record = record.as_dict() - return self.registry.Update(record, cred) + return self.registry().Update(record, cred) - def get_trusted_certs(self, opts, args): - """ - return uhe trusted certs at this interface - """ - trusted_certs = self.registry.get_trusted_certs() - for trusted_cert in trusted_certs: - gid = GID(string=trusted_cert) - gid.dump() - cert = Certificate(string=trusted_cert) - self.logger.debug('Sfi.get_trusted_certs -> %r'%cert.get_subject()) - return - - def aggregates(self, opts, args): - """ - return a list of details about known aggregates - """ - user_cred = self.get_user_cred().save_to_string(save_parents=True) - hrn = None - if args: - hrn = args[0] - - result = self.registry.get_aggregates(user_cred, hrn) - display_list(result) - return - - def registries(self, opts, args): - """ - return a list of details about known registries - """ - user_cred = self.get_user_cred().save_to_string(save_parents=True) - hrn = None - if args: - hrn = args[0] - result = self.registry.get_registries(user_cred, hrn) - display_list(result) - return - - + def remove(self, options, args): + "remove registry record by name (Remove)" + auth_cred = self.my_authority_credential_string() + if len(args)!=1: + self.print_help() + sys.exit(1) + hrn = args[0] + type = options.type + if type in ['all']: + type = '*' + return self.registry().Remove(hrn, auth_cred, type) + # ================================================================== # Slice-related commands # ================================================================== - def version(self, opts, args): - if opts.version_local: - version=version_core() - else: - if opts.version_registry: - server=self.registry - else: - server = self.server_proxy_from_opts(opts) - result = server.GetVersion() - version = ReturnValue.get_value(result) - for (k,v) in version.iteritems(): - print "%-20s: %s"%(k,v) - if opts.file: - save_variable_to_file(version, opts.file, opts.fileformat) - - # list instantiated slices - def slices(self, opts, args): - """ - list instantiated slices - """ - user_cred = self.get_user_cred().save_to_string(save_parents=True) - creds = [user_cred] - if opts.delegate: - delegated_cred = self.delegate_cred(user_cred, get_authority(self.authority)) + def slices(self, options, args): + "list instantiated slices (ListSlices) - returns urn's" + server = self.sliceapi() + # creds + creds = [self.my_credential_string] + if options.delegate: + delegated_cred = self.delegate_cred(self.my_credential_string, get_authority(self.authority)) creds.append(delegated_cred) - server = self.server_proxy_from_opts(opts) - call_args = [creds] - if self.server_supports_options_arg(server): - options = {'call_id': unique_call_id()} - call_args.append(options) - result = server.ListSlices(*call_args) + # options and call_id when supported + api_options = {} + api_options['call_id']=unique_call_id() + result = server.ListSlices(creds, *self.ois(server,api_options)) value = ReturnValue.get_value(result) display_list(value) return # show rspec for named slice - def resources(self, opts, args): - user_cred = self.get_user_cred().save_to_string(save_parents=True) - server = self.slicemgr - server = self.server_proxy_from_opts(opts) - - options = {'call_id': unique_call_id()} - #panos add info options - if opts.info: - options['info'] = opts.info - + def resources(self, options, args): + """ + with no arg, discover available resources, (ListResources) +or with an slice hrn, shows currently provisioned resources + """ + server = self.sliceapi() + + # set creds + creds = [] if args: - cred = self.get_slice_cred(args[0]).save_to_string(save_parents=True) - hrn = args[0] - options['geni_slice_urn'] = hrn_to_urn(hrn, 'slice') + creds.append(self.slice_credential_string(args[0])) else: - cred = user_cred - - creds = [cred] - if opts.delegate: - delegated_cred = self.delegate_cred(cred, get_authority(self.authority)) - creds.append(delegated_cred) - if opts.rspec_version: - version_manager = VersionManager() - server_version = self.get_cached_server_version(server) - if 'sfa' in server_version: - # just request the version the client wants - options['geni_rspec_version'] = version_manager.get_version(opts.rspec_version).to_dict() + creds.append(self.my_credential_string) + if options.delegate: + creds.append(self.delegate_cred(cred, get_authority(self.authority))) + + # V2 API + if self.server_supports_options_arg(server): + # with v2 everything goes in options inclusing the subject slice + api_options = {} + if args: + hrn = args[0] + api_options['geni_slice_urn'] = hrn_to_urn(hrn, 'slice') + if options.info: + api_options['info'] = options.info + if options.rspec_version: + version_manager = VersionManager() + server_version = self.get_cached_server_version(server) + if 'sfa' in server_version: + # just request the version the client wants + api_options['geni_rspec_version'] = version_manager.get_version(options.rspec_version).to_dict() + else: + # this must be a protogeni aggregate. We should request a v2 ad rspec + # regardless of what the client user requested + api_options['geni_rspec_version'] = version_manager.get_version('ProtoGENI 2').to_dict() else: - # this must be a protogeni aggregate. We should request a v2 ad rspec - # regardless of what the client user requested - options['geni_rspec_version'] = version_manager.get_version('ProtoGENI 2').to_dict() + api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3.0'} + api_options ['call_id'] = unique_call_id() + # the V2 form + result = server.ListResources (creds, api_options) + # V1 else: - options['geni_rspec_version'] = {'type': 'geni', 'version': '3.0'} - - call_args = [creds, options] - result = server.ListResources(*call_args) + # with an argument + if args: + hrn = args[0] + # xxx looks like we can pass a hrn and not a urn here ?? + # last arg. is a raw call_id when supported + result = server.ListResources (creds, hrn, *self.cis(server)) + else: + result = server.ListResources (creds, *self.cis(server)) value = ReturnValue.get_value(result) - if opts.file is None: - display_rspec(value, opts.format) + if options.file is None: + display_rspec(value, options.format) else: - save_rspec_to_file(value, opts.file) + save_rspec_to_file(value, options.file) return - # created named slice with given rspec - def create(self, opts, args): - server = self.server_proxy_from_opts(opts) - server_version = self.get_cached_server_version(server) + def create(self, options, args): + """ + create or update named slice with given rspec + """ + server = self.sliceapi() + + # xxx do we need to check usage (len(args)) ? + # slice urn slice_hrn = args[0] slice_urn = hrn_to_urn(slice_hrn, 'slice') - user_cred = self.get_user_cred() - slice_cred = self.get_slice_cred(slice_hrn).save_to_string(save_parents=True) - - if hasattr(opts, 'aggregate') and opts.aggregate: - delegated_cred = None - else: - # delegate the cred to the callers root authority - print>>sys.stderr, " \r\n \r\n \t SFI.PY get_authority(self.authority+'.slicemanager') %s self.authority %s slice_cred \t %s " %(get_authority(self.authority+'.slicemanager'), self.authority, slice_cred) - delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority+'.slicemanager')) - - #delegated_cred = self.delegate_cred(slice_cred, get_authority(slice_hrn)) - #creds.append(delegated_cred) + # credentials + creds = [self.slice_credential_string(slice_hrn)] + delegated_cred = None + server_version = self.get_cached_server_version(server) + if server_version.get('interface') == 'slicemgr': + # delegate our cred to the slice manager + # do not delegate cred to slicemgr...not working at the moment + pass + #if server_version.get('hrn'): + # delegated_cred = self.delegate_cred(slice_cred, server_version['hrn']) + #elif server_version.get('urn'): + # delegated_cred = self.delegate_cred(slice_cred, urn_to_hrn(server_version['urn'])) + + # rspec rspec_file = self.get_rspec_file(args[1]) rspec = open(rspec_file).read() + # users # need to pass along user keys to the aggregate. # users = [ # { urn: urn:publicid:IDN+emulab.net+user+alice # keys: [, ] # }] users = [] - print>>sys.stderr, " \r\n SFI.PY create slice_urn ", slice_urn - slice_records = self.registry.Resolve(slice_urn, [user_cred.save_to_string(save_parents=True)]) + slice_records = self.registry().Resolve(slice_urn, [self.my_credential_string]) if slice_records and 'researcher' in slice_records[0] and slice_records[0]['researcher']!=[]: slice_record = slice_records[0] user_hrns = slice_record['researcher'] user_urns = [hrn_to_urn(hrn, 'user') for hrn in user_hrns] - print>>sys.stderr, " \r\n SFI.PY create user_urns %s \r\n \t slice_records %s"%( user_urns,slice_records) - user_records = self.registry.Resolve(user_urns, [user_cred.save_to_string(save_parents=True)]) + user_records = self.registry().Resolve(user_urns, [self.my_credential_string]) if 'sfa' not in server_version: users = pg_users_arg(user_records) rspec = RSpec(rspec) rspec.filter({'component_manager_id': server_version['urn']}) rspec = RSpecConverter.to_pg_rspec(rspec.toxml(), content_type='request') - creds = [slice_cred] else: print >>sys.stderr, "\r\n \r\n \r\n WOOOOOO" users = sfa_users_arg(user_records, slice_record) - creds = [slice_cred] - if delegated_cred: - creds.append(delegated_cred) - call_args = [slice_urn, creds, rspec, users] - if self.server_supports_options_arg(server): - options = {'call_id': unique_call_id()} - call_args.append(options) - print>>sys.stderr, " \r\n SFI.PY create user" ,users - result = server.CreateSliver(*call_args) + + # do not append users, keys, or slice tags. Anything + # not contained in this request will be removed from the slice + + # CreateSliver has supported the options argument for a while now so it should + # be safe to assume this server support it + api_options = {} + api_options ['append'] = False + api_options ['call_id'] = unique_call_id() + + result = server.CreateSliver(slice_urn, creds, rspec, users, *self.ois(server, api_options)) value = ReturnValue.get_value(result) - if opts.file is None: + if options.file is None: print value else: - save_rspec_to_file (value, opts.file) + save_rspec_to_file (value, options.file) return value - # get a ticket for the specified slice - def get_ticket(self, opts, args): - slice_hrn, rspec_path = args[0], args[1] - slice_urn = hrn_to_urn(slice_hrn, 'slice') - user_cred = self.get_user_cred() - slice_cred = self.get_slice_cred(slice_hrn).save_to_string(save_parents=True) + def delete(self, options, args): + """ + delete named slice (DeleteSliver) + """ + server = self.sliceapi() + + # slice urn + slice_hrn = args[0] + slice_urn = hrn_to_urn(slice_hrn, 'slice') + + # creds + slice_cred = self.slice_credential_string(slice_hrn) creds = [slice_cred] - if opts.delegate: + if options.delegate: delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority)) creds.append(delegated_cred) - rspec_file = self.get_rspec_file(rspec_path) - rspec = open(rspec_file).read() - server = self.server_proxy_from_opts(opts) - ticket_string = server.GetTicket(slice_urn, creds, rspec, []) - file = os.path.join(self.options.sfi_dir, get_leaf(slice_hrn) + ".ticket") - self.logger.info("writing ticket to %s"%file) - ticket = SfaTicket(string=ticket_string) - ticket.save_to_file(filename=file, save_parents=True) - - def redeem_ticket(self, opts, args): - ticket_file = args[0] - - # get slice hrn from the ticket - # use this to get the right slice credential - ticket = SfaTicket(filename=ticket_file) - ticket.decode() - slice_hrn = ticket.gidObject.get_hrn() - slice_urn = hrn_to_urn(slice_hrn, 'slice') - #slice_hrn = ticket.attributes['slivers'][0]['hrn'] - user_cred = self.get_user_cred() - slice_cred = self.get_slice_cred(slice_hrn).save_to_string(save_parents=True) - - # get a list of node hostnames from the RSpec - tree = etree.parse(StringIO(ticket.rspec)) - root = tree.getroot() - hostnames = root.xpath("./network/site/node/hostname/text()") - # create an xmlrpc connection to the component manager at each of these - # components and gall redeem_ticket - connections = {} - for hostname in hostnames: - try: - self.logger.info("Calling redeem_ticket at %(hostname)s " % locals()) - server = self.server_proxy(hostname, CM_PORT, self.key_file, \ - self.cert_file, self.options.debug) - server.RedeemTicket(ticket.save_to_string(save_parents=True), slice_cred) - self.logger.info("Success") - except socket.gaierror: - self.logger.error("redeem_ticket failed: Component Manager not accepting requests") - except Exception, e: - self.logger.log_exc(e.message) - return - - # delete named slice - def delete(self, opts, args): + # options and call_id when supported + api_options = {} + api_options ['call_id'] = unique_call_id() + result = server.DeleteSliver(slice_urn, creds, *self.ois(server, api_options ) ) + # xxx no ReturnValue ?? + return result + + def status(self, options, args): + """ + retrieve slice status (SliverStatus) + """ + server = self.sliceapi() + + # slice urn slice_hrn = args[0] slice_urn = hrn_to_urn(slice_hrn, 'slice') - slice_cred = self.get_slice_cred(slice_hrn).save_to_string(save_parents=True) + + # creds + slice_cred = self.slice_credential_string(slice_hrn) creds = [slice_cred] - if opts.delegate: + if options.delegate: delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority)) creds.append(delegated_cred) - server = self.server_proxy_from_opts(opts) - call_args = [slice_urn, creds] - if self.server_supports_options_arg(server): - options = {'call_id': unique_call_id()} - call_args.append(options) - return server.DeleteSliver(*call_args) - - # start named slice - def start(self, opts, args): + + # options and call_id when supported + api_options = {} + api_options['call_id']=unique_call_id() + result = server.SliverStatus(slice_urn, creds, *self.ois(server,api_options)) + value = ReturnValue.get_value(result) + print value + if options.file: + save_variable_to_file(value, options.file, options.fileformat) + + def start(self, options, args): + """ + start named slice (Start) + """ + server = self.sliceapi() + + # the slice urn slice_hrn = args[0] slice_urn = hrn_to_urn(slice_hrn, 'slice') - slice_cred = self.get_slice_cred(args[0]).save_to_string(save_parents=True) + + # cred + slice_cred = self.slice_credential_string(args[0]) creds = [slice_cred] - if opts.delegate: + if options.delegate: delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority)) creds.append(delegated_cred) - server = self.server_proxy_from_opts(opts) + # xxx Thierry - does this not need an api_options as well ? return server.Start(slice_urn, creds) - # stop named slice - def stop(self, opts, args): + def stop(self, options, args): + """ + stop named slice (Stop) + """ + server = self.sliceapi() + # slice urn slice_hrn = args[0] slice_urn = hrn_to_urn(slice_hrn, 'slice') - slice_cred = self.get_slice_cred(args[0]).save_to_string(save_parents=True) + # cred + slice_cred = self.slice_credential_string(args[0]) creds = [slice_cred] - if opts.delegate: + if options.delegate: delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority)) creds.append(delegated_cred) - server = self.server_proxy_from_opts(opts) return server.Stop(slice_urn, creds) # reset named slice - def reset(self, opts, args): + def reset(self, options, args): + """ + reset named slice (reset_slice) + """ + server = self.sliceapi() + # slice urn slice_hrn = args[0] slice_urn = hrn_to_urn(slice_hrn, 'slice') - server = self.server_proxy_from_opts(opts) - slice_cred = self.get_slice_cred(args[0]).save_to_string(save_parents=True) + # cred + slice_cred = self.slice_credential_string(args[0]) creds = [slice_cred] - if opts.delegate: + if options.delegate: delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority)) creds.append(delegated_cred) return server.reset_slice(creds, slice_urn) - def renew(self, opts, args): + def renew(self, options, args): + """ + renew slice (RenewSliver) + """ + server = self.sliceapi() + # slice urn slice_hrn = args[0] slice_urn = hrn_to_urn(slice_hrn, 'slice') - server = self.server_proxy_from_opts(opts) - slice_cred = self.get_slice_cred(args[0]).save_to_string(save_parents=True) + # creds + slice_cred = self.slice_credential_string(args[0]) creds = [slice_cred] - if opts.delegate: + if options.delegate: delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority)) creds.append(delegated_cred) + # time time = args[1] - - call_args = [slice_urn, creds, time] - if self.server_supports_options_arg(server): - options = {'call_id': unique_call_id()} - call_args.append(options) - result = server.RenewSliver(*call_args) + # options and call_id when supported + api_options = {} + api_options['call_id']=unique_call_id() + result = server.RenewSliver(slice_urn, creds, time, *self.ois(server,api_options)) value = ReturnValue.get_value(result) return value - def status(self, opts, args): + def shutdown(self, options, args): + """ + shutdown named slice (Shutdown) + """ + server = self.sliceapi() + # slice urn slice_hrn = args[0] slice_urn = hrn_to_urn(slice_hrn, 'slice') - slice_cred = self.get_slice_cred(slice_hrn).save_to_string(save_parents=True) + # creds + slice_cred = self.slice_credential_string(slice_hrn) creds = [slice_cred] - if opts.delegate: + if options.delegate: delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority)) creds.append(delegated_cred) - server = self.server_proxy_from_opts(opts) - call_args = [slice_urn, creds] - if self.server_supports_options_arg(server): - options = {'call_id': unique_call_id()} - call_args.append(options) - result = server.SliverStatus(*call_args) - value = ReturnValue.get_value(result) - print value - if opts.file: - save_variable_to_file(value, opts.file, opts.fileformat) - + return server.Shutdown(slice_urn, creds) + - def shutdown(self, opts, args): - slice_hrn = args[0] - slice_urn = hrn_to_urn(slice_hrn, 'slice') - slice_cred = self.get_slice_cred(slice_hrn).save_to_string(save_parents=True) + def get_ticket(self, options, args): + """ + get a ticket for the specified slice + """ + server = self.sliceapi() + # slice urn + slice_hrn, rspec_path = args[0], args[1] + slice_urn = hrn_to_urn(slice_hrn, 'slice') + # creds + slice_cred = self.slice_credential_string(slice_hrn) creds = [slice_cred] - if opts.delegate: + if options.delegate: delegated_cred = self.delegate_cred(slice_cred, get_authority(self.authority)) creds.append(delegated_cred) - server = self.server_proxy_from_opts(opts) - return server.Shutdown(slice_urn, creds) - - def print_help (self): - self.sfi_parser.print_help() - self.cmd_parser.print_help() - - # - # Main: parse arguments and dispatch to command - # - def main(self): - self.sfi_parser = self.create_parser() - (options, args) = self.sfi_parser.parse_args() - self.options = options + # rspec + rspec_file = self.get_rspec_file(rspec_path) + rspec = open(rspec_file).read() + # options and call_id when supported + api_options = {} + api_options['call_id']=unique_call_id() + # get ticket at the server + ticket_string = server.GetTicket(slice_urn, creds, rspec, *self.ois(server,api_options)) + # save + file = os.path.join(self.options.sfi_dir, get_leaf(slice_hrn) + ".ticket") + self.logger.info("writing ticket to %s"%file) + ticket = SfaTicket(string=ticket_string) + ticket.save_to_file(filename=file, save_parents=True) - self.logger.setLevelFromOptVerbose(self.options.verbose) - if options.hashrequest: - self.hashrequest = True - - if len(args) <= 0: - self.logger.critical("No command given. Use -h for help.") - return -1 - - command = args[0] - self.cmd_parser = self.create_cmd_parser(command) - (cmd_opts, cmd_args) = self.cmd_parser.parse_args(args[1:]) + def redeem_ticket(self, options, args): + """ + Connects to nodes in a slice and redeems a ticket +(slice hrn is retrieved from the ticket) + """ + ticket_file = args[0] + + # get slice hrn from the ticket + # use this to get the right slice credential + ticket = SfaTicket(filename=ticket_file) + ticket.decode() + ticket_string = ticket.save_to_string(save_parents=True) - self.set_servers() - self.logger.info("Command=%s" % command) - if command in ("resources"): - self.logger.debug("resources cmd_opts %s" % cmd_opts.format) - elif command in ("list", "show", "remove"): - self.logger.debug("cmd_opts.type %s" % cmd_opts.type) - self.logger.debug('cmd_args %s' % cmd_args) + slice_hrn = ticket.gidObject.get_hrn() + slice_urn = hrn_to_urn(slice_hrn, 'slice') + #slice_hrn = ticket.attributes['slivers'][0]['hrn'] + slice_cred = self.slice_credential_string(slice_hrn) + + # get a list of node hostnames from the RSpec + tree = etree.parse(StringIO(ticket.rspec)) + root = tree.getroot() + hostnames = root.xpath("./network/site/node/hostname/text()") + + # create an xmlrpc connection to the component manager at each of these + # components and gall redeem_ticket + connections = {} + for hostname in hostnames: + try: + self.logger.info("Calling redeem_ticket at %(hostname)s " % locals()) + cm_url="http://%s:%s/"%(hostname,CM_PORT) + server = SfaServerProxy(cm_url, self.private_key, self.my_gid) + server = self.server_proxy(hostname, CM_PORT, self.private_key, + timeout=self.options.timeout, verbose=self.options.debug) + server.RedeemTicket(ticket_string, slice_cred) + self.logger.info("Success") + except socket.gaierror: + self.logger.error("redeem_ticket failed on %s: Component Manager not accepting requests"%hostname) + except Exception, e: + self.logger.log_exc(e.message) + return - try: - self.dispatch(command, cmd_opts, cmd_args) - except KeyError: - self.logger.critical ("Unknown command %s"%command) - raise + def create_gid(self, options, args): + """ + Create a GID (CreateGid) + """ + if len(args) < 1: + self.print_help() sys.exit(1) + target_hrn = args[0] + gid = self.registry().CreateGid(self.my_credential_string, target_hrn, self.bootstrap.my_gid_string()) + if options.file: + filename = options.file + else: + filename = os.sep.join([self.options.sfi_dir, '%s.gid' % target_hrn]) + self.logger.info("writing %s gid to %s" % (target_hrn, filename)) + GID(string=gid).save_to_file(filename) + + + def delegate(self, options, args): + """ + (locally) create delegate credential for use by given hrn + """ + delegee_hrn = args[0] + if options.delegate_user: + cred = self.delegate_cred(self.my_credential_string, delegee_hrn, 'user') + elif options.delegate_slice: + slice_cred = self.slice_credential_string(options.delegate_slice) + cred = self.delegate_cred(slice_cred, delegee_hrn, 'slice') + else: + self.logger.warning("Must specify either --user or --slice ") + return + delegated_cred = Credential(string=cred) + object_hrn = delegated_cred.get_gid_object().get_hrn() + if options.delegate_user: + dest_fn = os.path.join(self.options.sfi_dir, get_leaf(delegee_hrn) + "_" + + get_leaf(object_hrn) + ".cred") + elif options.delegate_slice: + dest_fn = os.path.join(self.options.sfi_dir, get_leaf(delegee_hrn) + "_slice_" + + get_leaf(object_hrn) + ".cred") + + delegated_cred.save_to_file(dest_fn, save_parents=True) + + self.logger.info("delegated credential for %s to %s and wrote to %s"%(object_hrn, delegee_hrn,dest_fn)) - return - -if __name__ == "__main__": - Sfi().main() + def get_trusted_certs(self, options, args): + """ + return uhe trusted certs at this interface (get_trusted_certs) + """ + trusted_certs = self.registry().get_trusted_certs() + for trusted_cert in trusted_certs: + gid = GID(string=trusted_cert) + gid.dump() + cert = Certificate(string=trusted_cert) + self.logger.debug('Sfi.get_trusted_certs -> %r'%cert.get_subject()) + return + diff --git a/sfa/client/getNodes.py b/sfa/clientbin/getNodes.py similarity index 100% rename from sfa/client/getNodes.py rename to sfa/clientbin/getNodes.py diff --git a/sfa/client/getRecord.py b/sfa/clientbin/getRecord.py similarity index 100% rename from sfa/client/getRecord.py rename to sfa/clientbin/getRecord.py diff --git a/sfa/client/setRecord.py b/sfa/clientbin/setRecord.py similarity index 100% rename from sfa/client/setRecord.py rename to sfa/clientbin/setRecord.py diff --git a/sfa/client/sfadump.py b/sfa/clientbin/sfadump.py similarity index 99% rename from sfa/client/sfadump.py rename to sfa/clientbin/sfadump.py index c45f7719..59dd38c5 100755 --- a/sfa/client/sfadump.py +++ b/sfa/clientbin/sfadump.py @@ -4,7 +4,6 @@ from __future__ import with_statement import sys import os, os.path import tempfile -import xmlrpclib from types import StringTypes, ListType from optparse import OptionParser diff --git a/sfa/clientbin/sfascan.py b/sfa/clientbin/sfascan.py new file mode 100755 index 00000000..641a0572 --- /dev/null +++ b/sfa/clientbin/sfascan.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +from sfa.client.sfascan import SfaScan + +if __name__ == '__main__': + SfaScan().main() diff --git a/sfa/clientbin/sfi.py b/sfa/clientbin/sfi.py new file mode 100755 index 00000000..bf80d59b --- /dev/null +++ b/sfa/clientbin/sfi.py @@ -0,0 +1,8 @@ +#! /usr/bin/env python + +# sfi -- slice-based facility interface + +from sfa.client.sfi import Sfi + +if __name__ == "__main__": + Sfi().main() diff --git a/sfa/client/sfiAddAttribute.py b/sfa/clientbin/sfiAddAttribute.py similarity index 100% rename from sfa/client/sfiAddAttribute.py rename to sfa/clientbin/sfiAddAttribute.py diff --git a/sfa/client/sfiAddLinks.py b/sfa/clientbin/sfiAddLinks.py similarity index 100% rename from sfa/client/sfiAddLinks.py rename to sfa/clientbin/sfiAddLinks.py diff --git a/sfa/client/sfiAddSliver.py b/sfa/clientbin/sfiAddSliver.py similarity index 100% rename from sfa/client/sfiAddSliver.py rename to sfa/clientbin/sfiAddSliver.py diff --git a/sfa/client/sfiDeleteAttribute.py b/sfa/clientbin/sfiDeleteAttribute.py similarity index 100% rename from sfa/client/sfiDeleteAttribute.py rename to sfa/clientbin/sfiDeleteAttribute.py diff --git a/sfa/client/sfiDeleteSliver.py b/sfa/clientbin/sfiDeleteSliver.py similarity index 100% rename from sfa/client/sfiDeleteSliver.py rename to sfa/clientbin/sfiDeleteSliver.py diff --git a/sfa/client/sfiListLinks.py b/sfa/clientbin/sfiListLinks.py similarity index 100% rename from sfa/client/sfiListLinks.py rename to sfa/clientbin/sfiListLinks.py diff --git a/sfa/client/sfiListNodes.py b/sfa/clientbin/sfiListNodes.py similarity index 100% rename from sfa/client/sfiListNodes.py rename to sfa/clientbin/sfiListNodes.py diff --git a/sfa/client/sfiListSlivers.py b/sfa/clientbin/sfiListSlivers.py similarity index 100% rename from sfa/client/sfiListSlivers.py rename to sfa/clientbin/sfiListSlivers.py diff --git a/sfa/examples/miniclient.py b/sfa/examples/miniclient.py new file mode 100755 index 00000000..1c625e07 --- /dev/null +++ b/sfa/examples/miniclient.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python + +# this is designed to use a totally empty new directory +# so we demonstrate how to bootstrap the whole thing + +# init logging on console +import logging +console = logging.StreamHandler() +logger=logging.getLogger('') +logger.addHandler(console) +logger.setLevel(logging.DEBUG) + +import uuid +def unique_call_id(): return uuid.uuid4().urn + +# use sys.argv to point to a completely fresh directory +import sys +args=sys.argv[1:] +if len(args)!=1: + print "Usage: %s directory"%sys.argv[0] + sys.exit(1) +dir=args[0] +logger.debug('sfaclientsample: Using directory %s'%dir) + +### + +# this uses a test sfa deployment at openlab +registry_url="http://sfa1.pl.sophia.inria.fr:12345/" +aggregate_url="http://sfa1.pl.sophia.inria.fr:12347/" +# this is where the private key sits - would be ~/.ssh/id_rsa in most cases +# but in this context, create this local file +# the tests key pair can be found in +# http://git.onelab.eu/?p=tests.git;a=blob;f=system/config_default.py +# search for public_key / private_key +private_key="miniclient-private-key" +# user hrn +user_hrn="pla.inri.fake-pi1" + +slice_hrn="pla.inri.slpl1" +# hrn_to_urn(slice_hrn,'slice') +slice_urn='urn:publicid:IDN+pla:inri+slice+slpl1' + +from sfa.client.sfaclientlib import SfaClientBootstrap + +bootstrap = SfaClientBootstrap (user_hrn, registry_url, dir=dir, logger=logger) +# install the private key in the client directory from 'private_key' +bootstrap.init_private_key_if_missing(private_key) + +def truncate(content, length=20, suffix='...'): + if isinstance (content, (int) ): return content + if isinstance (content, list): return truncate ( "%s"%content, length, suffix) + if len(content) <= length: + return content + else: + return content[:length+1]+ ' '+suffix + + +### issue a GetVersion call +### this assumes we've already somehow initialized the certificate +def get_version (url): + # make sure we have a self-signed cert + bootstrap.self_signed_cert() + server_proxy = bootstrap.server_proxy_simple(url) + server_version = server_proxy.GetVersion() + print "miniclient: GetVersion at %s returned:"%(url) + for (k,v) in server_version.iteritems(): print "miniclient: \tversion[%s]=%s"%(k,truncate(v)) + +# version_dict = {'type': 'SFA', 'version': '1', } + +version_dict = {'type':'ProtoGENI', 'version':'2'} + + +# ditto with list resources +def list_resources (): + bootstrap.bootstrap_my_gid() + credential = bootstrap.my_credential_string() + credentials = [ credential ] + options = {} + options [ 'geni_rspec_version' ] = version_dict + options [ 'call_id' ] = unique_call_id() + list_resources = bootstrap.server_proxy (aggregate_url).ListResources(credentials,options) + print "miniclient: ListResources at %s returned : %s"%(aggregate_url,truncate(list_resources)) + +def list_slice_resources (): + bootstrap.bootstrap_my_gid() + credential = bootstrap.slice_credential_string (slice_hrn) + credentials = [ credential ] + options = { } + options [ 'geni_rspec_version' ] = version_dict + options [ 'geni_slice_urn' ] = slice_urn + options [ 'call_id' ] = unique_call_id() + list_resources = bootstrap.server_proxy (aggregate_url).ListResources(credentials,options) + print "miniclient: ListResources at %s for slice %s returned : %s"%(aggregate_url,slice_urn,truncate(list_resources)) + + + +def main (): + get_version(registry_url) + get_version(aggregate_url) + list_resources() + list_slice_resources() + +main() diff --git a/sfa/generic/__init__.py b/sfa/generic/__init__.py index 363f3dd0..de1a9e14 100644 --- a/sfa/generic/__init__.py +++ b/sfa/generic/__init__.py @@ -35,7 +35,7 @@ class Generic: #mixed = flavour.capitalize() module_path="sfa.generic.%s"%flavour classname="%s"%flavour - logger.info("Generic.the_flavour with flavour=%s"%flavour) + logger.debug("Generic.the_flavour with flavour=%s"%flavour) try: module = __import__ (module_path, globals(), locals(), [classname]) return getattr(module, classname)(flavour,config) @@ -60,22 +60,25 @@ class Generic: if not 'interface' in kwargs: logger.critical("Generic.make_api: no interface found") api = self.api_class()(*args, **kwargs) - manager = self.make_manager(api.interface) + # xxx can probably drop support for managers implemented as modules + # which makes it a bit awkward + manager_class_or_module = self.make_manager(api.interface) driver = self.make_driver (api.config, api.interface) ### arrange stuff together # add a manager wrapper - manager_wrap = ManagerWrapper(manager,api.interface) + manager_wrap = ManagerWrapper(manager_class_or_module,api.interface,api.config) api.manager=manager_wrap # insert driver in manager - logger.info("Setting manager.driver, manager=%s"%manager) - manager.driver=driver + logger.debug("Setting manager.driver, manager=%s"%manager_class_or_module) + # xxx this should go into the object and not the class !?! + manager_class_or_module.driver=driver # add it in api as well for convenience api.driver=driver return api def make_manager (self, interface): """ - interface expected in ['registry', 'aggregate', 'slice', 'component'] + interface expected in ['registry', 'aggregate', 'slicemgr', 'component'] flavour is e.g. 'pl' or 'max' or whatever """ flavour = self.flavour @@ -83,9 +86,12 @@ class Generic: classname = "%s_manager_class"%interface try: - module = getattr(self,classname)() - logger.debug("%s : %s"%(message,module)) - return module + module_or_class = getattr(self,classname)() + logger.debug("%s : %s"%(message,module_or_class)) + # this gets passed to ManagerWrapper that will call the class constructor + # if it's a class, or use the module as is if it's a module + # so bottom line is, don't try the constructor here + return module_or_class except: logger.log_exc_critical(message) diff --git a/sfa/generic/max.py b/sfa/generic/max.py new file mode 100644 index 00000000..8920ae7c --- /dev/null +++ b/sfa/generic/max.py @@ -0,0 +1,21 @@ +# an example of how to plugin the max aggregate manager with the flavour model +# might need to be tested +# +from sfa.generic.pl import pl + +import sfa.managers.aggregate_manager_max + +class max (pl): + +# the max flavour behaves like pl, except for +# the aggregate + def aggregate_manager_class (self) : + return sfa.managers.aggregate_manager_max.AggregateManagerMax + +# I believe the component stuff is not implemented + def component_manager_class (self): + return None + def component_driver_class (self): + return None + + diff --git a/sfa/importer/sfa-import-plc.py b/sfa/importer/sfa-import-plc.py index a9915525..d41bbd4c 100755 --- a/sfa/importer/sfa-import-plc.py +++ b/sfa/importer/sfa-import-plc.py @@ -18,10 +18,9 @@ import os import getopt import sys -from sfa.util.xrn import get_leaf, get_authority -from sfa.util.plxrn import hostname_to_hrn, slicename_to_hrn, email_to_hrn, hrn_to_pl_slicename from sfa.util.config import Config -from sfa.util.xrn import Xrn +from sfa.util.xrn import Xrn, get_leaf, get_authority +from sfa.util.plxrn import hostname_to_hrn, slicename_to_hrn, email_to_hrn, hrn_to_pl_slicename from sfa.storage.table import SfaTable diff --git a/sfa/managers/aggregate_manager.py b/sfa/managers/aggregate_manager.py index 2fbdf3d3..89559204 100644 --- a/sfa/managers/aggregate_manager.py +++ b/sfa/managers/aggregate_manager.py @@ -1,165 +1,59 @@ -import datetime -import time -import sys - -from sfa.util.sfalogging import logger -from sfa.util.faults import RecordNotFound, SliverDoesNotExist -from sfa.util.xrn import get_authority, hrn_to_urn, urn_to_hrn, Xrn, urn_to_sliver_id -from sfa.util.plxrn import slicename_to_hrn, hrn_to_pl_slicename from sfa.util.version import version_core -from sfa.util.sfatime import utcparse +from sfa.util.xrn import Xrn from sfa.util.callids import Callids -from sfa.trust.sfaticket import SfaTicket -from sfa.trust.credential import Credential -from sfa.rspecs.version_manager import VersionManager -from sfa.rspecs.rspec import RSpec - -from sfa.server.sfaapi import SfaApi - -import sfa.plc.peers as peers -from sfa.plc.aggregate import Aggregate -from sfa.plc.slices import Slices - class AggregateManager: - def __init__ (self): - # xxx Thierry : caching at the aggregate level sounds wrong... - self.caching=True - #self.caching=False + def __init__ (self, config): pass - def GetVersion(self, api, options={}): + # essentially a union of the core version, the generic version (this code) and + # whatever the driver needs to expose + def GetVersion(self, api, options): - version_manager = VersionManager() - ad_rspec_versions = [] - request_rspec_versions = [] - for rspec_version in version_manager.versions: - if rspec_version.content_type in ['*', 'ad']: - ad_rspec_versions.append(rspec_version.to_dict()) - if rspec_version.content_type in ['*', 'request']: - request_rspec_versions.append(rspec_version.to_dict()) xrn=Xrn(api.hrn) - version_more = {'interface':'aggregate', - 'sfa': 2, - 'geni_api': api.config.SFA_AGGREGATE_API_VERSION, - 'testbed':'myplc', - 'hrn':xrn.get_hrn(), - 'geni_request_rspec_versions': request_rspec_versions, - 'geni_ad_rspec_versions': ad_rspec_versions, - } - return version_core(version_more) - - def _get_registry_objects(self, slice_xrn, creds, users): - """ - - """ - hrn, _ = urn_to_hrn(slice_xrn) - - hrn_auth = get_authority(hrn) - - # Build up objects that an SFA registry would return if SFA - # could contact the slice's registry directly - reg_objects = None - - if users: - # dont allow special characters in the site login base - #only_alphanumeric = re.compile('[^a-zA-Z0-9]+') - #login_base = only_alphanumeric.sub('', hrn_auth[:20]).lower() - slicename = hrn_to_pl_slicename(hrn) - login_base = slicename.split('_')[0] - reg_objects = {} - site = {} - site['site_id'] = 0 - site['name'] = 'geni.%s' % login_base - site['enabled'] = True - site['max_slices'] = 100 - - # Note: - # Is it okay if this login base is the same as one already at this myplc site? - # Do we need uniqueness? Should use hrn_auth instead of just the leaf perhaps? - site['login_base'] = login_base - site['abbreviated_name'] = login_base - site['max_slivers'] = 1000 - reg_objects['site'] = site - - slice = {} - - # get_expiration always returns a normalized datetime - no need to utcparse - extime = Credential(string=creds[0]).get_expiration() - # If the expiration time is > 60 days from now, set the expiration time to 60 days from now - if extime > datetime.datetime.utcnow() + datetime.timedelta(days=60): - extime = datetime.datetime.utcnow() + datetime.timedelta(days=60) - slice['expires'] = int(time.mktime(extime.timetuple())) - slice['hrn'] = hrn - slice['name'] = hrn_to_pl_slicename(hrn) - slice['url'] = hrn - slice['description'] = hrn - slice['pointer'] = 0 - reg_objects['slice_record'] = slice - - reg_objects['users'] = {} - for user in users: - user['key_ids'] = [] - hrn, _ = urn_to_hrn(user['urn']) - user['email'] = hrn_to_pl_slicename(hrn) + "@geni.net" - user['first_name'] = hrn - user['last_name'] = hrn - reg_objects['users'][user['email']] = user - - return reg_objects + version = version_core() + version_generic = {'interface':'aggregate', + 'sfa': 2, + 'geni_api': 2, + 'hrn':xrn.get_hrn(), + 'urn':xrn.get_urn(), + } + version.update(version_generic) + testbed_version = self.driver.aggregate_version() + version.update(testbed_version) + return version + + def ListSlices(self, api, creds, options): + call_id = options.get('call_id') + if Callids().already_handled(call_id): return [] + return self.driver.list_slices (creds, options) + + def ListResources(self, api, creds, options): + call_id = options.get('call_id') + if Callids().already_handled(call_id): return "" + + # get slice's hrn from options + slice_xrn = options.get('geni_slice_urn', None) + # pass None if no slice is specified + if not slice_xrn: + slice_hrn, slice_urn = None, None + else: + xrn = Xrn(slice_xrn) + slice_urn=xrn.get_urn() + slice_hrn=xrn.get_hrn() + + return self.driver.list_resources (slice_urn, slice_hrn, creds, options) - def SliverStatus(self, api, slice_xrn, creds, options={}): + def SliverStatus (self, api, xrn, creds, options): call_id = options.get('call_id') if Callids().already_handled(call_id): return {} - (hrn, _) = urn_to_hrn(slice_xrn) - # find out where this slice is currently running - slicename = hrn_to_pl_slicename(hrn) - - slices = api.driver.GetSlices([slicename], ['slice_id', 'node_ids','person_ids','name','expires']) - if len(slices) == 0: - raise Exception("Slice %s not found (used %s as slicename internally)" % (slice_xrn, slicename)) - slice = slices[0] - - # report about the local nodes only - nodes = api.driver.GetNodes({'node_id':slice['node_ids'],'peer_id':None}, - ['node_id', 'hostname', 'site_id', 'boot_state', 'last_contact']) - site_ids = [node['site_id'] for node in nodes] - - result = {} - top_level_status = 'unknown' - if nodes: - top_level_status = 'ready' - slice_urn = Xrn(slice_xrn, 'slice').get_urn() - result['geni_urn'] = slice_urn - result['pl_login'] = slice['name'] - result['pl_expires'] = datetime.datetime.fromtimestamp(slice['expires']).ctime() - - resources = [] - for node in nodes: - res = {} - res['pl_hostname'] = node['hostname'] - res['pl_boot_state'] = node['boot_state'] - res['pl_last_contact'] = node['last_contact'] - if node['last_contact'] is not None: - res['pl_last_contact'] = datetime.datetime.fromtimestamp(node['last_contact']).ctime() - sliver_id = urn_to_sliver_id(slice_urn, slice['slice_id'], node['node_id']) - res['geni_urn'] = sliver_id - if node['boot_state'] == 'boot': - res['geni_status'] = 'ready' - else: - res['geni_status'] = 'failed' - top_level_status = 'failed' - - res['geni_error'] = '' - - resources.append(res) - - result['geni_status'] = top_level_status - result['geni_resources'] = resources - return result + xrn = Xrn(xrn) + slice_urn=xrn.get_urn() + slice_hrn=xrn.get_hrn() + return self.driver.sliver_status (slice_urn, slice_hrn) - def CreateSliver(self, api, slice_xrn, creds, rspec_string, users, options={}): + def CreateSliver(self, api, xrn, creds, rspec_string, users, options): """ Create the sliver[s] (slice) at this aggregate. Verify HRN and initialize the slice record in PLC if necessary. @@ -167,232 +61,54 @@ class AggregateManager: call_id = options.get('call_id') if Callids().already_handled(call_id): return "" - aggregate = Aggregate(api) - slices = Slices(api) - (hrn, _) = urn_to_hrn(slice_xrn) - peer = slices.get_peer(hrn) - sfa_peer = slices.get_sfa_peer(hrn) - slice_record=None - if users: - slice_record = users[0].get('slice_record', {}) - - # parse rspec - rspec = RSpec(rspec_string) - requested_attributes = rspec.version.get_slice_attributes() - - # ensure site record exists - site = slices.verify_site(hrn, slice_record, peer, sfa_peer) - # ensure slice record exists - slice = slices.verify_slice(hrn, slice_record, peer, sfa_peer) - # ensure person records exists - persons = slices.verify_persons(hrn, slice, users, peer, sfa_peer) - # ensure slice attributes exists - slices.verify_slice_attributes(slice, requested_attributes) - - # add/remove slice from nodes - requested_slivers = [node.get('component_name') for node in rspec.version.get_nodes_with_slivers()] - slices.verify_slice_nodes(slice, requested_slivers, peer) - - # add/remove links links - slices.verify_slice_links(slice, rspec.version.get_link_requests(), aggregate) - - # handle MyPLC peer association. - # only used by plc and ple. - slices.handle_peer(site, slice, persons, peer) - - return aggregate.get_rspec(slice_xrn=slice_xrn, version=rspec.version) - + xrn = Xrn(xrn) + slice_urn=xrn.get_urn() + slice_hrn=xrn.get_hrn() + + return self.driver.create_sliver (slice_urn, slice_hrn, creds, rspec_string, users, options) - def RenewSliver(self, api, xrn, creds, expiration_time, options={}): + def DeleteSliver(self, api, xrn, creds, options): + call_id = options.get('call_id') + if Callids().already_handled(call_id): return True + + xrn = Xrn(xrn) + slice_urn=xrn.get_urn() + slice_hrn=xrn.get_hrn() + return self.driver.delete_sliver (slice_urn, slice_hrn, creds, options) + + def RenewSliver(self, api, xrn, creds, expiration_time, options): call_id = options.get('call_id') if Callids().already_handled(call_id): return True - (hrn, _) = urn_to_hrn(xrn) - slicename = hrn_to_pl_slicename(hrn) - slices = api.driver.GetSlices({'name': slicename}, ['slice_id']) - if not slices: - raise RecordNotFound(hrn) - slice = slices[0] - requested_time = utcparse(expiration_time) - record = {'expires': int(time.mktime(requested_time.timetuple()))} - try: - api.driver.UpdateSlice(slice['slice_id'], record) - return True - except: - return False + + xrn = Xrn(xrn) + slice_urn=xrn.get_urn() + slice_hrn=xrn.get_hrn() + return self.driver.renew_sliver (slice_urn, slice_hrn, creds, expiration_time, options) + ### these methods could use an options extension for at least call_id def start_slice(self, api, xrn, creds): - (hrn, _) = urn_to_hrn(xrn) - slicename = hrn_to_pl_slicename(hrn) - slices = api.driver.GetSlices({'name': slicename}, ['slice_id']) - if not slices: - raise RecordNotFound(hrn) - slice_id = slices[0]['slice_id'] - slice_tags = api.driver.GetSliceTags({'slice_id': slice_id, 'tagname': 'enabled'}, ['slice_tag_id']) - # just remove the tag if it exists - if slice_tags: - api.driver.DeleteSliceTag(slice_tags[0]['slice_tag_id']) - - return 1 + xrn = Xrn(xrn) + slice_urn=xrn.get_urn() + slice_hrn=xrn.get_hrn() + return self.driver.start_slice (slice_urn, slice_hrn, creds) def stop_slice(self, api, xrn, creds): - hrn, _ = urn_to_hrn(xrn) - slicename = hrn_to_pl_slicename(hrn) - slices = api.driver.GetSlices({'name': slicename}, ['slice_id']) - if not slices: - raise RecordNotFound(hrn) - slice_id = slices[0]['slice_id'] - slice_tags = api.driver.GetSliceTags({'slice_id': slice_id, 'tagname': 'enabled'}) - if not slice_tags: - api.driver.AddSliceTag(slice_id, 'enabled', '0') - elif slice_tags[0]['value'] != "0": - tag_id = slice_tags[0]['slice_tag_id'] - api.driver.UpdateSliceTag(tag_id, '0') - return 1 - + xrn = Xrn(xrn) + slice_urn=xrn.get_urn() + slice_hrn=xrn.get_hrn() + return self.driver.stop_slice (slice_urn, slice_hrn, creds) + def reset_slice(self, api, xrn): - # XX not implemented at this interface - return 1 - - def DeleteSliver(self, api, xrn, creds, options={}): - call_id = options.get('call_id') - if Callids().already_handled(call_id): return "" - (hrn, _) = urn_to_hrn(xrn) - slicename = hrn_to_pl_slicename(hrn) - slices = api.driver.GetSlices({'name': slicename}) - if not slices: - return 1 - slice = slices[0] - - # determine if this is a peer slice - peer = peers.get_peer(api, hrn) - try: - if peer: - api.driver.UnBindObjectFromPeer('slice', slice['slice_id'], peer) - api.driver.DeleteSliceFromNodes(slicename, slice['node_ids']) - finally: - if peer: - api.driver.BindObjectToPeer('slice', slice['slice_id'], peer, slice['peer_slice_id']) - return 1 - - def ListSlices(self, api, creds, options={}): - call_id = options.get('call_id') - if Callids().already_handled(call_id): return [] - # look in cache first - if self.caching and api.cache: - slices = api.cache.get('slices') - if slices: - return slices - - # get data from db - slices = api.driver.GetSlices({'peer_id': None}, ['name']) - slice_hrns = [slicename_to_hrn(api.hrn, slice['name']) for slice in slices] - slice_urns = [hrn_to_urn(slice_hrn, 'slice') for slice_hrn in slice_hrns] - - # cache the result - if self.caching and api.cache: - api.cache.add('slices', slice_urns) - - return slice_urns - - def ListResources(self, api, creds, options={}): - call_id = options.get('call_id') - if Callids().already_handled(call_id): return "" - # get slice's hrn from options - xrn = options.get('geni_slice_urn', None) - cached = options.get('cached', True) - (hrn, _) = urn_to_hrn(xrn) - - version_manager = VersionManager() - # get the rspec's return format from options - rspec_version = version_manager.get_version(options.get('geni_rspec_version')) - version_string = "rspec_%s" % (rspec_version) - - #panos adding the info option to the caching key (can be improved) - if options.get('info'): - version_string = version_string + "_"+options.get('info', 'default') - - # look in cache first - if self.caching and api.cache and not xrn and cached: - rspec = api.cache.get(version_string) - if rspec: - api.logger.info("aggregate.ListResources: returning cached value for hrn %s"%hrn) - return rspec - - #panos: passing user-defined options - #print "manager options = ",options - aggregate = Aggregate(api) - rspec = aggregate.get_rspec(slice_xrn=xrn, version=rspec_version, options=options) - - # cache the result - if self.caching and api.cache and not xrn: - api.cache.add(version_string, rspec) - - return rspec - - - def GetTicket(self, api, xrn, creds, rspec, users, options={}): - - (slice_hrn, _) = urn_to_hrn(xrn) - slices = Slices(api) - peer = slices.get_peer(slice_hrn) - sfa_peer = slices.get_sfa_peer(slice_hrn) - - # get the slice record - credential = api.getCredential() - interface = api.registries[api.hrn] - registry = api.server_proxy(interface, credential) - records = registry.Resolve(xrn, credential) - - # make sure we get a local slice record - record = None - for tmp_record in records: - if tmp_record['type'] == 'slice' and \ - not tmp_record['peer_authority']: - #Error (E0602, GetTicket): Undefined variable 'SliceRecord' - record = SliceRecord(dict=tmp_record) - if not record: - raise RecordNotFound(slice_hrn) - - # similar to CreateSliver, we must verify that the required records exist - # at this aggregate before we can issue a ticket - # parse rspec - rspec = RSpec(rspec_string) - requested_attributes = rspec.version.get_slice_attributes() - - # ensure site record exists - site = slices.verify_site(hrn, slice_record, peer, sfa_peer) - # ensure slice record exists - slice = slices.verify_slice(hrn, slice_record, peer, sfa_peer) - # ensure person records exists - persons = slices.verify_persons(hrn, slice, users, peer, sfa_peer) - # ensure slice attributes exists - slices.verify_slice_attributes(slice, requested_attributes) - - # get sliver info - slivers = slices.get_slivers(slice_hrn) - - if not slivers: - raise SliverDoesNotExist(slice_hrn) - - # get initscripts - initscripts = [] - data = { - 'timestamp': int(time.time()), - 'initscripts': initscripts, - 'slivers': slivers - } - - # create the ticket - object_gid = record.get_gid_object() - new_ticket = SfaTicket(subject = object_gid.get_subject()) - new_ticket.set_gid_caller(api.auth.client_gid) - new_ticket.set_gid_object(object_gid) - new_ticket.set_issuer(key=api.key, subject=api.hrn) - new_ticket.set_pubkey(object_gid.get_pubkey()) - new_ticket.set_attributes(data) - new_ticket.set_rspec(rspec) - #new_ticket.set_parent(api.auth.hierarchy.get_auth_ticket(auth_hrn)) - new_ticket.encode() - new_ticket.sign() + xrn = Xrn(xrn) + slice_urn=xrn.get_urn() + slice_hrn=xrn.get_hrn() + return self.driver.reset_slice (slice_urn, slice_hrn) + + def GetTicket(self, api, xrn, creds, rspec, users, options): - return new_ticket.save_to_string(save_parents=True) + xrn = Xrn(xrn) + slice_urn=xrn.get_urn() + slice_hrn=xrn.get_hrn() + + return self.driver.get_ticket (slice_urn, slice_hrn, creds, rspec, options) + diff --git a/sfa/managers/aggregate_manager_eucalyptus.py b/sfa/managers/aggregate_manager_eucalyptus.py index 55e14ce6..552f5446 100644 --- a/sfa/managers/aggregate_manager_eucalyptus.py +++ b/sfa/managers/aggregate_manager_eucalyptus.py @@ -23,13 +23,16 @@ from sfa.util.callids import Callids #from sfa.util.sfalogging import logger from sfa.util.version import version_core from sfa.trust.credential import Credential -from sfa.server.sfaapi import SfaApi -from sfa.plc.aggregate import Aggregate -from sfa.plc.slices import Slice, Slices +from sfa.plc.plaggregate import PlAggregate +# No Slice symbol in there +#from sfa.plc.plslices import Slice, Slices +from sfa.plc.plslices import PlSlices from sfa.rspecs.version_manager import VersionManager from sfa.rspecs.rspec import RSpec # not sure what this used to be nor where it is now defined #from sfa.rspecs.sfa_rspec import sfa_rspec_version +# most likely this should now be +#from sfa.rspecs.version_manager import VersionManager ## # Meta data of an instance. @@ -283,7 +286,7 @@ class AggregateManagerEucalyptus: _inited=False # the init_server mechanism has vanished - def __init__ (self): + def __init__ (self, config): if AggregateManagerEucalyptus._inited: return AggregateManagerEucalyptus.init_server() @@ -444,7 +447,7 @@ class AggregateManagerEucalyptus: port=eucaPort, path=srvPath) - def ListResources(api, creds, options={}): + def ListResources(api, creds, options): call_id = options.get('call_id') if Callids().already_handled(call_id): return "" # get slice's hrn from options @@ -541,15 +544,15 @@ class AggregateManagerEucalyptus: """ Hook called via 'sfi.py create' """ - def CreateSliver(api, slice_xrn, creds, xml, users, options={}): + def CreateSliver(api, slice_xrn, creds, xml, users, options): call_id = options.get('call_id') if Callids().already_handled(call_id): return "" logger = logging.getLogger('EucaAggregate') logger.debug("In CreateSliver") - aggregate = Aggregate(api) - slices = Slices(api) + aggregate = PlAggregate(self.driver) + slices = PlSlices(self.driver) (hrn, type) = urn_to_hrn(slice_xrn) peer = slices.get_peer(hrn) sfa_peer = slices.get_sfa_peer(hrn) @@ -679,7 +682,7 @@ class AggregateManagerEucalyptus: f.write("%s %s %s\n" % (instId, ipaddr, hrn)) f.close() - def GetVersion(api, options={}): + def GetVersion(api, options): version_manager = VersionManager() ad_rspec_versions = [] @@ -692,7 +695,7 @@ class AggregateManagerEucalyptus: xrn=Xrn(api.hrn) version_more = {'interface':'aggregate', 'sfa': 1, - 'geni_api': api.config.SFA_AGGREGATE_API_VERSION, + 'geni_api': '2', 'testbed':'myplc', 'hrn':xrn.get_hrn(), 'geni_request_rspec_versions': request_rspec_versions, diff --git a/sfa/managers/aggregate_manager_max.py b/sfa/managers/aggregate_manager_max.py index 91130bc4..03005228 100644 --- a/sfa/managers/aggregate_manager_max.py +++ b/sfa/managers/aggregate_manager_max.py @@ -14,10 +14,13 @@ from sfa.util.xrn import urn_to_hrn, hrn_to_urn, Xrn from sfa.managers.aggregate_manager import AggregateManager -from sfa.plc.slices import Slices +from sfa.plc.plslices import PlSlices class AggregateManagerMax (AggregateManager): + def __init__ (self, config): + pass + RSPEC_TMP_FILE_PREFIX = "/tmp/max_rspec" # execute shell command and return both exit code and text output @@ -81,11 +84,71 @@ class AggregateManagerMax (AggregateManager): if indx1!=-1 and indx2>indx1: xml = text[indx1:indx2+len(tag)+2] return xml + + # formerly in aggregate_manager.py but got unused in there... + def _get_registry_objects(self, slice_xrn, creds, users): + """ + + """ + hrn, _ = urn_to_hrn(slice_xrn) + + #hrn_auth = get_authority(hrn) + + # Build up objects that an SFA registry would return if SFA + # could contact the slice's registry directly + reg_objects = None + + if users: + # dont allow special characters in the site login base + #only_alphanumeric = re.compile('[^a-zA-Z0-9]+') + #login_base = only_alphanumeric.sub('', hrn_auth[:20]).lower() + slicename = hrn_to_pl_slicename(hrn) + login_base = slicename.split('_')[0] + reg_objects = {} + site = {} + site['site_id'] = 0 + site['name'] = 'geni.%s' % login_base + site['enabled'] = True + site['max_slices'] = 100 + + # Note: + # Is it okay if this login base is the same as one already at this myplc site? + # Do we need uniqueness? Should use hrn_auth instead of just the leaf perhaps? + site['login_base'] = login_base + site['abbreviated_name'] = login_base + site['max_slivers'] = 1000 + reg_objects['site'] = site + + slice = {} + + # get_expiration always returns a normalized datetime - no need to utcparse + extime = Credential(string=creds[0]).get_expiration() + # If the expiration time is > 60 days from now, set the expiration time to 60 days from now + if extime > datetime.datetime.utcnow() + datetime.timedelta(days=60): + extime = datetime.datetime.utcnow() + datetime.timedelta(days=60) + slice['expires'] = int(time.mktime(extime.timetuple())) + slice['hrn'] = hrn + slice['name'] = hrn_to_pl_slicename(hrn) + slice['url'] = hrn + slice['description'] = hrn + slice['pointer'] = 0 + reg_objects['slice_record'] = slice + + reg_objects['users'] = {} + for user in users: + user['key_ids'] = [] + hrn, _ = urn_to_hrn(user['urn']) + user['email'] = hrn_to_pl_slicename(hrn) + "@geni.net" + user['first_name'] = hrn + user['last_name'] = hrn + reg_objects['users'][user['email']] = user + + return reg_objects def prepare_slice(self, api, slice_xrn, creds, users): reg_objects = self._get_registry_objects(slice_xrn, creds, users) (hrn, type) = urn_to_hrn(slice_xrn) - slices = Slices(api) + slices = PlSlices(self.driver) peer = slices.get_peer(hrn) sfa_peer = slices.get_sfa_peer(hrn) slice_record=None @@ -214,12 +277,12 @@ class AggregateManagerMax (AggregateManager): ### GENI AM API Methods - def SliverStatus(self, api, slice_xrn, creds, options={}): + def SliverStatus(self, api, slice_xrn, creds, options): call_id = options.get('call_id') if Callids().already_handled(call_id): return {} return self.slice_status(api, slice_xrn, creds) - def CreateSliver(self, api, slice_xrn, creds, rspec_string, users, options={}): + def CreateSliver(self, api, slice_xrn, creds, rspec_string, users, options): call_id = options.get('call_id') if Callids().already_handled(call_id): return "" #TODO: create real CreateSliver response rspec @@ -229,13 +292,13 @@ class AggregateManagerMax (AggregateManager): else: return " Error! " - def DeleteSliver(self, api, xrn, creds, options={}): + def DeleteSliver(self, api, xrn, creds, options): call_id = options.get('call_id') if Callids().already_handled(call_id): return "" return self.delete_slice(api, xrn, creds) # no caching - def ListResources(self, api, creds, options={}): + def ListResources(self, api, creds, options): call_id = options.get('call_id') if Callids().already_handled(call_id): return "" # version_string = "rspec_%s" % (rspec_version.get_version_name()) diff --git a/sfa/managers/component_manager_pl.py b/sfa/managers/component_manager_pl.py index 69929714..96304790 100644 --- a/sfa/managers/component_manager_pl.py +++ b/sfa/managers/component_manager_pl.py @@ -5,7 +5,7 @@ from sfa.util.plxrn import PlXrn from sfa.trust.sfaticket import SfaTicket from sfa.util.version import version_core -def GetVersion(api): +def GetVersion(api, options): return version_core({'interface':'component', 'testbed':'myplc'}) diff --git a/sfa/managers/driver.py b/sfa/managers/driver.py index b12cd710..17358cfc 100644 --- a/sfa/managers/driver.py +++ b/sfa/managers/driver.py @@ -5,7 +5,13 @@ import sys class Driver: - def __init__ (self): pass + def __init__ (self, config): + # this is the hrn attached to the running server + self.hrn = config.SFA_INTERFACE_HRN + + ######################################## + ########## registry oriented + ######################################## # redefine this if you want to check again records # when running GetCredential @@ -18,6 +24,10 @@ class Driver: # after looking up the sfa db, we wish to be able to display # testbed-specific info as well # this at minima should fill in the 'researcher' field for slice records + # as this information is then used to compute rights + # roadmap: there is an intention to redesign the SFA database so as to clear up + # this constraint, based on the principle that SFA should not rely on the + # testbed database to perform such a core operation (i.e. getting rights right) def augment_records_with_testbed_info (self, sfa_records): print >>sys.stderr, " \r\n \r\n DRIVER.PY augment_records_with_testbed_info sfa_records ",sfa_records return sfa_records @@ -26,7 +36,6 @@ class Driver: # expected retcod 'pointer' # 'pointer' is typically an int db id, that makes sense in the testbed environment # -1 if this feature is not relevant - # here type will be 'authority' def register (self, sfa_record, hrn, pub_key) : return -1 @@ -51,3 +60,58 @@ class Driver: # should anything be passed back to the caller in this case ? def update (self, old_sfa_record, new_sfa_record, hrn, new_key): return True + + ######################################## + ########## aggregate oriented + ######################################## + + # a name for identifying the kind of testbed + def testbed_name (self): return "undefined" + + # a dictionary that gets appended to the generic answer to GetVersion + # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory + def aggregate_version (self): return {} + + # the answer to ListSlices, a list of slice urns + def list_slices (self, creds, options): + return [] + + # answer to ListResources + # first 2 args are None in case of resource discovery + # expected : rspec (xml string) + def list_resources (self, slice_urn, slice_hrn, creds, options): + return "dummy Driver.list_resources needs to be redefined" + + # the answer to SliverStatus on a given slice + def sliver_status (self, slice_urn, slice_hrn): return {} + + # the answer to CreateSliver on a given slice + # expected to return a valid rspec + # identical to ListResources after the slice was modified + def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, users, options): + return "dummy Driver.create_sliver needs to be redefined" + + # the answer to DeleteSliver on a given slice + def delete_sliver (self, slice_urn, slice_hrn, creds, options): + return "dummy Driver.delete_sliver needs to be redefined" + + # the answer to RenewSliver + # expected to return a boolean to indicate success + def renew_sliver (self, slice_urn, slice_hrn, creds, expiration_time, options): + return False + + # the answer to start_slice/stop_slice + # 1 means success, otherwise raise exception + def start_slice (self, slice_urn, slice_xrn, creds): + return 1 + def stop_slice (self, slice_urn, slice_xrn, creds): + return 1 + # somehow this one does not have creds - not implemented in PL anyways + def reset_slice (self, slice_urn, slice_xrn, creds): + return 1 + + # the answer to GetTicket + # expected is a ticket, i.e. a certificate, as a string + def get_ticket (self, slice_urn, slice_xrn, creds, rspec, options): + return "dummy Driver.get_ticket needs to be redefined" + diff --git a/sfa/managers/managerwrapper.py b/sfa/managers/managerwrapper.py index 86907e79..58a0527b 100644 --- a/sfa/managers/managerwrapper.py +++ b/sfa/managers/managerwrapper.py @@ -15,14 +15,14 @@ class ManagerWrapper: is not implemented by a libarary and will generally be more helpful than the standard AttributeError """ - def __init__(self, manager, interface): + def __init__(self, manager, interface, config): if isinstance (manager, ModuleType): # old-fashioned module implementation self.manager = manager elif isinstance (manager, ClassType): # create an instance; we don't pass the api in argument as it is passed # to the actual method calls anyway - self.manager = manager() + self.manager = manager(config) else: raise SfaAPIError,"Argument to ManagerWrapper must be a module or class" self.interface = interface diff --git a/sfa/managers/registry_manager.py b/sfa/managers/registry_manager.py index 93d2ea8f..d29aafef 100644 --- a/sfa/managers/registry_manager.py +++ b/sfa/managers/registry_manager.py @@ -23,11 +23,11 @@ from sfa.storage.table import SfaTable class RegistryManager: - def __init__ (self): pass + def __init__ (self, config): pass # The GENI GetVersion call - def GetVersion(self, api): - peers = dict ( [ (hrn,interface._ServerProxy__host) for (hrn,interface) in api.registries.iteritems() + def GetVersion(self, api, options): + peers = dict ( [ (hrn,interface.get_url()) for (hrn,interface) in api.registries.iteritems() if hrn != api.hrn]) xrn=Xrn(api.hrn) return version_core({'interface':'registry', @@ -283,8 +283,7 @@ class RegistryManager: gid = auth_info.get_gid_object() record.set_gid(gid.save_to_string(save_parents=True)) - # update testbed-specific data f needed - logger.info("Getting driver from manager=%s"%self) + # update testbed-specific data if needed pointer = self.driver.register (record, hrn, pub_key) record.set_pointer(pointer) diff --git a/sfa/managers/slice_manager.py b/sfa/managers/slice_manager.py index 76ea6d27..be6cc7e2 100644 --- a/sfa/managers/slice_manager.py +++ b/sfa/managers/slice_manager.py @@ -12,48 +12,32 @@ from sfa.util.sfalogging import logger from sfa.util.xrn import Xrn, urn_to_hrn from sfa.util.version import version_core from sfa.util.callids import Callids +from sfa.util.cache import Cache + from sfa.server.threadmanager import ThreadManager + from sfa.rspecs.rspec_converter import RSpecConverter from sfa.rspecs.version_manager import VersionManager from sfa.rspecs.rspec import RSpec + from sfa.client.client_helper import sfa_to_pg_users_arg from sfa.client.return_value import ReturnValue class SliceManager: - def __init__ (self): - # self.caching=False - self.caching=True + + # the cache instance is a class member so it survives across incoming requests + cache = None + + def __init__ (self, config): + self.cache=None + if config.SFA_SM_CACHING: + if SliceManager.cache is None: + SliceManager.cache = Cache() + self.cache = SliceManager.cache - def _options_supported(self, api, server): - """ - Returns true if server support the optional call_id arg, false otherwise. - """ - server_version = api.get_cached_server_version(server) - - if 'sfa' in server_version: - code_tag = server_version['code_tag'] - code_tag_parts = code_tag.split("-") - - version_parts = code_tag_parts[0].split(".") - major, minor = version_parts[0:2] - rev = code_tag_parts[1] - if int(major) >= 1: - if int(minor) >= 2: - return True - return False - - # we have specialized xmlrpclib.ServerProxy to remember the input url - # OTOH it's not clear if we're only dealing with XMLRPCServerProxy instances - def get_serverproxy_url (self, server): - try: - return server.get_url() - except: - logger.warning("GetVersion, falling back to xmlrpclib.ServerProxy internals") - return server._ServerProxy__host + server._ServerProxy__handler - - def GetVersion(self, api, options={}): + def GetVersion(self, api, options): # peers explicitly in aggregates.xml - peers =dict ([ (peername,self.get_serverproxy_url(v)) for (peername,v) in api.aggregates.iteritems() + peers =dict ([ (peername,interface.get_url()) for (peername,interface) in api.aggregates.iteritems() if peername != api.hrn]) version_manager = VersionManager() ad_rspec_versions = [] @@ -65,8 +49,8 @@ class SliceManager: request_rspec_versions.append(rspec_version.to_dict()) xrn=Xrn(api.hrn, 'authority+sa') version_more = {'interface':'slicemgr', - 'sfa': 1, - 'geni_api': api.config.SFA_AGGREGATE_API_VERSION, + 'sfa': 2, + 'geni_api': 2, 'hrn' : xrn.get_hrn(), 'urn' : xrn.get_urn(), 'peers': peers, @@ -76,7 +60,7 @@ class SliceManager: sm_version=version_core(version_more) # local aggregate if present needs to have localhost resolved if api.hrn in api.aggregates: - local_am_url=self.get_serverproxy_url(api.aggregates[api.hrn]) + local_am_url=api.aggregates[api.hrn].get_url() sm_version['peers'][api.hrn]=local_am_url.replace('localhost',sm_version['hostname']) return sm_version @@ -96,7 +80,8 @@ class SliceManager: else: stats_tag = rspec.xml.root.add_element("statistics", call=callname) - stat_tag = stats_tag.add_element("aggregate", name=str(aggname), elapsed=str(elapsed), status=str(status)) + stat_tag = stats_tag.add_element("aggregate", name=str(aggname), + elapsed=str(elapsed), status=str(status)) if exc_info: exc_tag = stat_tag.add_element("exc_info", name=str(exc_info[1])) @@ -107,32 +92,33 @@ class SliceManager: # formats the traceback as a set of xml elements tb = traceback.extract_tb(exc_info[2]) for item in tb: - exc_frame = exc_tag.add_element("tb_frame", filename=str(item[0]), line=str(item[1]), func=str(item[2]), code=str(item[3])) + exc_frame = exc_tag.add_element("tb_frame", filename=str(item[0]), + line=str(item[1]), func=str(item[2]), code=str(item[3])) except Exception, e: logger.warn("add_slicemgr_stat failed on %s: %s" %(aggname, str(e))) - def ListResources(self, api, creds, options={}): + def ListResources(self, api, creds, options): + call_id = options.get('call_id') + if Callids().already_handled(call_id): return "" + version_manager = VersionManager() - def _ListResources(aggregate, server, credential, opts={}): - - my_opts = copy(opts) - args = [credential, my_opts] + + def _ListResources(aggregate, server, credential, options): + forward_options = copy(options) tStart = time.time() try: version = api.get_cached_server_version(server) # force ProtoGENI aggregates to give us a v2 RSpec if 'sfa' in version.keys(): - my_opts['rspec_version'] = version_manager.get_version('SFA 1').to_dict() + forward_options['rspec_version'] = version_manager.get_version('SFA 1').to_dict() else: - my_opts['rspec_version'] = version_manager.get_version('ProtoGENI 2').to_dict() - rspec = server.ListResources(*args) + forward_options['rspec_version'] = version_manager.get_version('ProtoGENI 2').to_dict() + rspec = server.ListResources(credential, forward_options) return {"aggregate": aggregate, "rspec": rspec, "elapsed": time.time()-tStart, "status": "success"} except Exception, e: api.logger.log_exc("ListResources failed at %s" %(server.url)) return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception", "exc_info": sys.exc_info()} - call_id = options.get('call_id') - if Callids().already_handled(call_id): return "" # get slice's hrn from options xrn = options.get('geni_slice_urn', '') @@ -145,9 +131,10 @@ class SliceManager: version_string = "rspec_%s" % (rspec_version) # look in cache first - if self.caching and api.cache and not xrn: - rspec = api.cache.get(version_string) + if self.cache and not xrn: + rspec = self.cache.get(version_string) if rspec: + api.logger.debug("SliceManager.ListResources returns cached advertisement") return rspec # get the callers hrn @@ -179,7 +166,8 @@ class SliceManager: result_version = version_manager._get_version(rspec_version.type, rspec_version.version, 'ad') rspec = RSpec(version=result_version) for result in results: - self.add_slicemgr_stat(rspec, "ListResources", result["aggregate"], result["elapsed"], result["status"], result.get("exc_info",None)) + self.add_slicemgr_stat(rspec, "ListResources", result["aggregate"], result["elapsed"], + result["status"], result.get("exc_info",None)) if result["status"]=="success": try: rspec.version.merge(ReturnValue.get_value(result["rspec"])) @@ -187,16 +175,19 @@ class SliceManager: api.logger.log_exc("SM.ListResources: Failed to merge aggregate rspec") # cache the result - if self.caching and api.cache and not xrn: - api.cache.add(version_string, rspec.toxml()) + if self.cache and not xrn: + api.logger.debug("SliceManager.ListResources caches advertisement") + self.cache.add(version_string, rspec.toxml()) return rspec.toxml() - - - def CreateSliver(self, api, xrn, creds, rspec_str, users, options={}): + + + def CreateSliver(self, api, xrn, creds, rspec_str, users, options): + call_id = options.get('call_id') + if Callids().already_handled(call_id): return "" version_manager = VersionManager() - def _CreateSliver(aggregate, server, xrn, credential, rspec, users, options={}): + def _CreateSliver(aggregate, server, xrn, credential, rspec, users, options): tStart = time.time() try: # Need to call GetVersion at an aggregate to determine the supported @@ -211,17 +202,12 @@ class SliceManager: rspec.filter(filter) rspec = rspec.toxml() requested_users = sfa_to_pg_users_arg(users) - args = [xrn, credential, rspec, requested_users] - if self._options_supported(api, server): - args.append(options) - rspec = server.CreateSliver(*args) + rspec = server.CreateSliver(xrn, credential, rspec, requested_users, options) return {"aggregate": aggregate, "rspec": rspec, "elapsed": time.time()-tStart, "status": "success"} except: logger.log_exc('Something wrong in _CreateSliver with URL %s'%server.url) return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception", "exc_info": sys.exc_info()} - call_id = options.get('call_id') - if Callids().already_handled(call_id): return "" # Validate the RSpec against PlanetLab's schema --disabled for now # The schema used here needs to aggregate the PL and VINI schemas # schema = "/var/www/html/schemas/pl.rng" @@ -252,13 +238,14 @@ class SliceManager: interface = api.aggregates[aggregate] server = api.server_proxy(interface, cred) # Just send entire RSpec to each aggregate - threads.run(_CreateSliver, aggregate, server, xrn, [cred], rspec.toxml(), users, call_id) + threads.run(_CreateSliver, aggregate, server, xrn, [cred], rspec.toxml(), users, options) results = threads.get_results() manifest_version = version_manager._get_version(rspec.version.type, rspec.version.version, 'manifest') result_rspec = RSpec(version=manifest_version) for result in results: - self.add_slicemgr_stat(result_rspec, "CreateSliver", result["aggregate"], result["elapsed"], result["status"], result.get("exc_info",None)) + self.add_slicemgr_stat(result_rspec, "CreateSliver", result["aggregate"], result["elapsed"], + result["status"], result.get("exc_info",None)) if result["status"]=="success": try: result_rspec.version.merge(ReturnValue.get_value(result["rspec"])) @@ -266,16 +253,12 @@ class SliceManager: api.logger.log_exc("SM.CreateSliver: Failed to merge aggregate rspec") return result_rspec.toxml() - def RenewSliver(self, api, xrn, creds, expiration_time, options={}): - def _RenewSliver(server, xrn, creds, expiration_time, options={}): - server_version = api.get_cached_server_version(server) - args = [xrn, creds, expiration_time] - if self._options_supported(api, server): - args.append(options) - return server.RenewSliver(*args) - + def RenewSliver(self, api, xrn, creds, expiration_time, options): call_id = options.get('call_id') if Callids().already_handled(call_id): return True + + def _RenewSliver(server, xrn, creds, expiration_time, options): + return server.RenewSliver(xrn, creds, expiration_time, options) (hrn, type) = urn_to_hrn(xrn) # get the callers hrn @@ -294,21 +277,18 @@ class SliceManager: continue interface = api.aggregates[aggregate] server = api.server_proxy(interface, cred) - threads.run(_RenewSliver, server, xrn, [cred], expiration_time, call_id) + threads.run(_RenewSliver, server, xrn, [cred], expiration_time, options) # 'and' the results results = [ReturnValue.get_value(result) for result in threads.get_results()] return reduce (lambda x,y: x and y, results , True) - def DeleteSliver(self, api, xrn, creds, options={}): - def _DeleteSliver(server, xrn, creds, options={}): - server_version = api.get_cached_server_version(server) - args = [xrn, creds] - if self._options_supported(api, server): - args.append(options) - return server.DeleteSliver(*args) - + def DeleteSliver(self, api, xrn, creds, options): call_id = options.get('call_id') if Callids().already_handled(call_id): return "" + + def _DeleteSliver(server, xrn, creds, options): + return server.DeleteSliver(xrn, creds, options) + (hrn, type) = urn_to_hrn(xrn) # get the callers hrn valid_cred = api.auth.checkCredentials(creds, 'deletesliver', hrn)[0] @@ -326,19 +306,15 @@ class SliceManager: continue interface = api.aggregates[aggregate] server = api.server_proxy(interface, cred) - threads.run(_DeleteSliver, server, xrn, [cred], call_id) + threads.run(_DeleteSliver, server, xrn, [cred], options) threads.get_results() return 1 # first draft at a merging SliverStatus - def SliverStatus(self, api, slice_xrn, creds, options={}): - def _SliverStatus(server, xrn, creds, options={}): - server_version = api.get_cached_server_version(server) - args = [xrn, creds] - if self._options_supported(api, server): - args.append(options) - return server.SliverStatus(*args) + def SliverStatus(self, api, slice_xrn, creds, options): + def _SliverStatus(server, xrn, creds, options): + return server.SliverStatus(xrn, creds, options) call_id = options.get('call_id') if Callids().already_handled(call_id): return {} @@ -350,10 +326,10 @@ class SliceManager: for aggregate in api.aggregates: interface = api.aggregates[aggregate] server = api.server_proxy(interface, cred) - threads.run (_SliverStatus, server, slice_xrn, [cred], call_id) + threads.run (_SliverStatus, server, slice_xrn, [cred], options) results = [ReturnValue.get_value(result) for result in threads.get_results()] - # get rid of any void result - e.g. when call_id was hit where by convention we return {} + # get rid of any void result - e.g. when call_id was hit, where by convention we return {} results = [ result for result in results if result and result['geni_resources']] # do not try to combine if there's no result @@ -374,21 +350,19 @@ class SliceManager: return overall - def ListSlices(self, api, creds, options={}): - def _ListSlices(server, creds, options={}): - server_version = api.get_cached_server_version(server) - args = [creds] - if self._options_supported(api, server): - args.append(options) - return server.ListSlices(*args) - + def ListSlices(self, api, creds, options): call_id = options.get('call_id') if Callids().already_handled(call_id): return [] + def _ListSlices(server, creds, options): + return server.ListSlices(creds, options) + # look in cache first - if self.caching and api.cache: - slices = api.cache.get('slices') + # xxx is this really frequent enough that it is worth being cached ? + if self.cache: + slices = self.cache.get('slices') if slices: + api.logger.debug("SliceManager.ListSlices returns from cache") return slices # get the callers hrn @@ -417,13 +391,14 @@ class SliceManager: slices.extend(result) # cache the result - if self.caching and api.cache: - api.cache.add('slices', slices) + if self.cache: + api.logger.debug("SliceManager.ListSlices caches value") + self.cache.add('slices', slices) return slices - def GetTicket(self, api, xrn, creds, rspec, users, options={}): + def GetTicket(self, api, xrn, creds, rspec, users, options): slice_hrn, type = urn_to_hrn(xrn) # get the netspecs contained within the clients rspec aggregate_rspecs = {} @@ -443,6 +418,7 @@ class SliceManager: cred = api.getCredential() threads = ThreadManager() for (aggregate, aggregate_rspec) in aggregate_rspecs.iteritems(): + # xxx sounds like using call_id here would be safer # prevent infinite loop. Dont send request back to caller # unless the caller is the aggregate's SM if caller_hrn == aggregate and aggregate != api.hrn: diff --git a/sfa/methods/CreateSliver.py b/sfa/methods/CreateSliver.py index 8a325f2a..b28d1a53 100644 --- a/sfa/methods/CreateSliver.py +++ b/sfa/methods/CreateSliver.py @@ -1,12 +1,10 @@ -from sfa.util.faults import SfaInvalidArgument +from sfa.util.faults import SfaInvalidArgument, InvalidRSpec from sfa.util.xrn import urn_to_hrn from sfa.util.method import Method from sfa.util.sfatablesRuntime import run_sfatables import sys from sfa.trust.credential import Credential - from sfa.storage.parameter import Parameter, Mixed - from sfa.rspecs.rspec import RSpec class CreateSliver(Method): @@ -32,7 +30,7 @@ class CreateSliver(Method): ] returns = Parameter(str, "Allocated RSpec") - def call(self, slice_xrn, creds, rspec, users, options={}): + def call(self, slice_xrn, creds, rspec, users, options): hrn, type = urn_to_hrn(slice_xrn) self.api.logger.info("interface: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, hrn, self.name)) @@ -53,10 +51,8 @@ class CreateSliver(Method): chain_name = 'FORWARD-INCOMING' self.api.logger.debug("CreateSliver: sfatables on chain %s"%chain_name) rspec = run_sfatables(chain_name, hrn, origin_hrn, rspec) - slivers = RSpec(rspec).version.get_nodes_with_slivers() - print >>sys.stderr, " \r\n \r\n Createsliver.py call users : ", users - if slivers: - result = self.api.manager.CreateSliver(self.api, slice_xrn, creds, rspec, users, options) - else: - result = rspec + slivers = RSpec(rspec).version.get_nodes_with_slivers() + if not slivers: + raise InvalidRSpec("Missing or element. Request rspec must explicitly allocate slivers") + result = self.api.manager.CreateSliver(self.api, slice_xrn, creds, rspec, users, options) return result diff --git a/sfa/methods/DeleteSliver.py b/sfa/methods/DeleteSliver.py index 6be2bc4e..697b6d3e 100644 --- a/sfa/methods/DeleteSliver.py +++ b/sfa/methods/DeleteSliver.py @@ -24,7 +24,7 @@ class DeleteSliver(Method): returns = Parameter(int, "1 if successful") - def call(self, xrn, creds, options={}): + def call(self, xrn, creds, options): (hrn, type) = urn_to_hrn(xrn) valid_creds = self.api.auth.checkCredentials(creds, 'deletesliver', hrn) diff --git a/sfa/methods/GetTicket.py b/sfa/methods/GetTicket.py index 2223e373..54978e7e 100644 --- a/sfa/methods/GetTicket.py +++ b/sfa/methods/GetTicket.py @@ -36,7 +36,7 @@ class GetTicket(Method): returns = Parameter(str, "String representation of a ticket object") - def call(self, xrn, creds, rspec, users, options={}): + def call(self, xrn, creds, rspec, users, options): hrn, type = urn_to_hrn(xrn) # Find the valid credentials valid_creds = self.api.auth.checkCredentials(creds, 'getticket', hrn) diff --git a/sfa/methods/GetVersion.py b/sfa/methods/GetVersion.py index 9dc586f1..cb682e44 100644 --- a/sfa/methods/GetVersion.py +++ b/sfa/methods/GetVersion.py @@ -9,9 +9,12 @@ class GetVersion(Method): @return version """ interfaces = ['registry','aggregate', 'slicemgr', 'component'] - accepts = [Parameter(dict, "Options")] + accepts = [ + Parameter(dict, "Options") + ] returns = Parameter(dict, "Version information") + # API v2 specifies options is optional, so.. def call(self, options={}): self.api.logger.info("interface: %s\tmethod-name: %s" % (self.api.interface, self.name)) return self.api.manager.GetVersion(self.api, options) diff --git a/sfa/methods/ListResources.py b/sfa/methods/ListResources.py index 386b7f2f..41676e69 100644 --- a/sfa/methods/ListResources.py +++ b/sfa/methods/ListResources.py @@ -23,12 +23,15 @@ class ListResources(Method): ] returns = Parameter(str, "List of resources") - def call(self, creds, options={}): + def call(self, creds, options): self.api.logger.info("interface: %s\tmethod-name: %s" % (self.api.interface, self.name)) # client must specify a version if not options.get('geni_rspec_version'): - raise SfaInvalidArgument('Must specify an rspec version option. geni_rspec_version cannot be null') + if options.get('rspec_version'): + options['geni_rspec_version'] = options['rspec_version'] + else: + raise SfaInvalidArgument('Must specify an rspec version option. geni_rspec_version cannot be null') # get slice's hrn from options xrn = options.get('geni_slice_urn', '') diff --git a/sfa/methods/ListSlices.py b/sfa/methods/ListSlices.py index e14c9b6d..b1967776 100644 --- a/sfa/methods/ListSlices.py +++ b/sfa/methods/ListSlices.py @@ -22,7 +22,7 @@ class ListSlices(Method): returns = Parameter(list, "List of slice names") - def call(self, creds, options={}): + def call(self, creds, options): valid_creds = self.api.auth.checkCredentials(creds, 'listslices') #log the call diff --git a/sfa/methods/RenewSliver.py b/sfa/methods/RenewSliver.py index a380d992..c503b557 100644 --- a/sfa/methods/RenewSliver.py +++ b/sfa/methods/RenewSliver.py @@ -26,7 +26,7 @@ class RenewSliver(Method): ] returns = Parameter(bool, "Success or Failure") - def call(self, slice_xrn, creds, expiration_time, options={}): + def call(self, slice_xrn, creds, expiration_time, options): (hrn, type) = urn_to_hrn(slice_xrn) diff --git a/sfa/methods/SliverStatus.py b/sfa/methods/SliverStatus.py index f722c7cd..deb79983 100644 --- a/sfa/methods/SliverStatus.py +++ b/sfa/methods/SliverStatus.py @@ -19,7 +19,7 @@ class SliverStatus(Method): ] returns = Parameter(dict, "Status details") - def call(self, slice_xrn, creds, options={}): + def call(self, slice_xrn, creds, options): hrn, type = urn_to_hrn(slice_xrn) valid_creds = self.api.auth.checkCredentials(creds, 'sliverstatus', hrn) diff --git a/sfa/methods/UpdateSliver.py b/sfa/methods/UpdateSliver.py index 3302b56d..72910d74 100644 --- a/sfa/methods/UpdateSliver.py +++ b/sfa/methods/UpdateSliver.py @@ -27,7 +27,7 @@ class UpdateSliver(CreateSliver): - def call(self, slice_xrn, creds, rspec, users, options={}): + def call(self, slice_xrn, creds, rspec, users, options): return CreateSliver.call(self, slice_xrn, creds, rspec, users, options) diff --git a/sfa/plc/peers.py b/sfa/plc/peers.py index b659ea9a..c256c7b5 100644 --- a/sfa/plc/peers.py +++ b/sfa/plc/peers.py @@ -1,8 +1,8 @@ from sfa.util.xrn import get_authority from types import StringTypes -def get_peer(api, hrn): - # Becaues of myplc federation, we first need to determine if this +def get_peer(pldriver, hrn): + # Because of myplc native federation, we first need to determine if this # slice belongs to out local plc or a myplc peer. We will assume it # is a local site, unless we find out otherwise peer = None @@ -13,7 +13,7 @@ def get_peer(api, hrn): # get this site's authority (sfa root authority or sub authority) site_authority = get_authority(slice_authority).lower() # check if we are already peered with this site_authority, if so - peers = api.driver.GetPeers( {}, ['peer_id', 'peername', 'shortname', 'hrn_root']) + peers = pldriver.GetPeers( {}, ['peer_id', 'peername', 'shortname', 'hrn_root']) for peer_record in peers: names = [name.lower() for name in peer_record.values() if isinstance(name, StringTypes)] if site_authority in names: @@ -22,14 +22,14 @@ def get_peer(api, hrn): return peer -def get_sfa_peer(api, hrn): - # return the authority for this hrn or None if we are the authority - sfa_peer = None - slice_authority = get_authority(hrn) - site_authority = get_authority(slice_authority) - - if site_authority != api.hrn: - sfa_peer = site_authority - - return sfa_peer +#def get_sfa_peer(pldriver, hrn): +# # return the authority for this hrn or None if we are the authority +# sfa_peer = None +# slice_authority = get_authority(hrn) +# site_authority = get_authority(slice_authority) +# +# if site_authority != pldriver.hrn: +# sfa_peer = site_authority +# +# return sfa_peer diff --git a/sfa/plc/aggregate.py b/sfa/plc/plaggregate.py similarity index 81% rename from sfa/plc/aggregate.py rename to sfa/plc/plaggregate.py index 0eff6cca..4cc2c55e 100644 --- a/sfa/plc/aggregate.py +++ b/sfa/plc/plaggregate.py @@ -1,6 +1,7 @@ #!/usr/bin/python -from sfa.util.xrn import hrn_to_urn, urn_to_hrn, urn_to_sliver_id -from sfa.util.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename +from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn, urn_to_sliver_id +from sfa.util.sfatime import epochparse +from sfa.util.sfalogging import logger from sfa.rspecs.rspec import RSpec from sfa.rspecs.elements.hardware_type import HardwareType @@ -12,29 +13,27 @@ from sfa.rspecs.elements.location import Location from sfa.rspecs.elements.interface import Interface from sfa.rspecs.elements.services import Services from sfa.rspecs.elements.pltag import PLTag -from sfa.util.topology import Topology from sfa.rspecs.version_manager import VersionManager + +from sfa.util.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename from sfa.plc.vlink import get_tc_rate -from sfa.util.sfatime import epochparse +from sfa.plc.topology import Topology -class Aggregate: - api = None - #panos new user options variable - user_options = {} +class PlAggregate: - def __init__(self, api): - self.api = api - + def __init__(self, driver): + self.driver = driver + def get_sites(self, filter={}): sites = {} - for site in self.api.driver.GetSites(filter): + for site in self.driver.GetSites(filter): sites[site['site_id']] = site return sites def get_interfaces(self, filter={}): interfaces = {} - for interface in self.api.driver.GetInterfaces(filter): + for interface in self.driver.GetInterfaces(filter): iface = Interface() if interface['bwlimit']: interface['bwlimit'] = str(int(interface['bwlimit'])/1000) @@ -54,8 +53,8 @@ class Aggregate: site1 = sites[site_id1] site2 = sites[site_id2] # get hrns - site1_hrn = self.api.hrn + '.' + site1['login_base'] - site2_hrn = self.api.hrn + '.' + site2['login_base'] + site1_hrn = self.driver.hrn + '.' + site1['login_base'] + site2_hrn = self.driver.hrn + '.' + site2['login_base'] for s1_node_id in site1['node_ids']: for s2_node_id in site2['node_ids']: @@ -65,9 +64,9 @@ class Aggregate: node2 = nodes[s2_node_id] # set interfaces # just get first interface of the first node - if1_xrn = PlXrn(auth=self.api.hrn, interface='node%s:eth0' % (node1['node_id'])) + if1_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node1['node_id'])) if1_ipv4 = interfaces[node1['interface_ids'][0]]['ip'] - if2_xrn = PlXrn(auth=self.api.hrn, interface='node%s:eth0' % (node2['node_id'])) + if2_xrn = PlXrn(auth=self.driver.hrn, interface='node%s:eth0' % (node2['node_id'])) if2_ipv4 = interfaces[node2['interface_ids'][0]]['ip'] if1 = Interface({'component_id': if1_xrn.urn, 'ipv4': if1_ipv4} ) @@ -78,22 +77,22 @@ class Aggregate: link['interface1'] = if1 link['interface2'] = if2 link['component_name'] = "%s:%s" % (site1['login_base'], site2['login_base']) - link['component_id'] = PlXrn(auth=self.api.hrn, interface=link['component_name']).get_urn() - link['component_manager_id'] = hrn_to_urn(self.api.hrn, 'authority+am') + link['component_id'] = PlXrn(auth=self.driver.hrn, interface=link['component_name']).get_urn() + link['component_manager_id'] = hrn_to_urn(self.driver.hrn, 'authority+am') links.append(link) return links def get_node_tags(self, filter={}): node_tags = {} - for node_tag in self.api.driver.GetNodeTags(filter): + for node_tag in self.driver.GetNodeTags(filter): node_tags[node_tag['node_tag_id']] = node_tag return node_tags def get_pl_initscripts(self, filter={}): pl_initscripts = {} filter.update({'enabled': True}) - for initscript in self.api.driver.GetInitScripts(filter): + for initscript in self.driver.GetInitScripts(filter): pl_initscripts[initscript['initscript_id']] = initscript return pl_initscripts @@ -109,7 +108,7 @@ class Aggregate: slice_urn = hrn_to_urn(slice_xrn, 'slice') slice_hrn, _ = urn_to_hrn(slice_xrn) slice_name = hrn_to_pl_slicename(slice_hrn) - slices = self.api.driver.GetSlices(slice_name) + slices = self.driver.GetSlices(slice_name) if not slices: return (slice, slivers) slice = slices[0] @@ -123,7 +122,7 @@ class Aggregate: slivers[node_id]= sliver # sort sliver attributes by node id - tags = self.api.driver.GetSliceTags({'slice_tag_id': slice['slice_tag_ids']}) + tags = self.driver.GetSliceTags({'slice_tag_id': slice['slice_tag_ids']}) for tag in tags: # most likely a default/global sliver attribute (node_id == None) if tag['node_id'] not in slivers: @@ -147,7 +146,7 @@ class Aggregate: filter['boot_state'] = 'boot' filter.update({'peer_id': None}) - nodes = self.api.driver.GetNodes(filter) + nodes = self.driver.GetNodes(filter) site_ids = [] interface_ids = [] @@ -180,10 +179,10 @@ class Aggregate: # xxx how to retrieve site['login_base'] site_id=node['site_id'] site=sites_dict[site_id] - rspec_node['component_id'] = hostname_to_urn(self.api.hrn, site['login_base'], node['hostname']) + rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname']) rspec_node['component_name'] = node['hostname'] - rspec_node['component_manager_id'] = self.api.hrn - rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.api.hrn, site['login_base']), 'authority+sa') + rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn() + rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa') rspec_node['boot_state'] = node['boot_state'] rspec_node['exclusive'] = 'False' rspec_node['hardware_types']= [HardwareType({'name': 'plab-pc'}), @@ -202,7 +201,11 @@ class Aggregate: for if_id in node['interface_ids']: interface = Interface(interfaces[if_id]) interface['ipv4'] = interface['ip'] - interface['component_id'] = PlXrn(auth=self.api.hrn, interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn() + interface['component_id'] = PlXrn(auth=self.driver.hrn, + interface='node%s:eth%s' % (node['node_id'], if_count)).get_urn() + # interfaces in the manifest need a client id + if slice: + interface['client_id'] = "%s:%s" % (node['node_id'], if_id) rspec_node['interfaces'].append(interface) if_count+=1 @@ -242,10 +245,12 @@ class Aggregate: rspec.version.add_links(links) # add sliver defaults - default_sliver_attribs = slivers.get(None, []) - for sliver_attrib in default_sliver_attribs: - rspec.version.add_default_sliver_attribute(sliver_attrib['name'], sliver_attrib['value']) - + default_sliver = slivers.get(None, []) + if default_sliver: + default_sliver_attribs = default_sliver.get('tags', []) + for attrib in default_sliver_attribs: + logger.info(attrib) + rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value']) return rspec.toxml() diff --git a/sfa/plc/plcomponentdriver.py b/sfa/plc/plcomponentdriver.py index 07a655c6..991cdde2 100644 --- a/sfa/plc/plcomponentdriver.py +++ b/sfa/plc/plcomponentdriver.py @@ -1,7 +1,7 @@ import os import tempfile -import sfa.client.xmlrpcprotocol as xmlrpcprotocol +from sfa.client.sfaserverproxy import SfaServerProxy from sfa.plc.nodemanager import NodeManager from sfa.trust.credential import Credential @@ -33,7 +33,7 @@ class PlComponentDriver: addr, port = self.config.SFA_REGISTRY_HOST, self.config.SFA_REGISTRY_PORT url = "http://%(addr)s:%(port)s" % locals() ### xxx this would require access to the api... - server = xmlrpcprotocol.server_proxy(url, self.key_file, self.cert_file) + server = SfaServerProxy(url, self.key_file, self.cert_file) return server def get_node_key(self): diff --git a/sfa/plc/pldriver.py b/sfa/plc/pldriver.py index d5d5bf9a..5ce3b710 100644 --- a/sfa/plc/pldriver.py +++ b/sfa/plc/pldriver.py @@ -1,17 +1,33 @@ +import time +import datetime # -from sfa.util.faults import MissingSfaInfo, UnknownSfaType +from sfa.util.faults import MissingSfaInfo, UnknownSfaType, \ + RecordNotFound, SfaNotImplemented, SliverDoesNotExist + from sfa.util.sfalogging import logger from sfa.util.defaultdict import defaultdict -from sfa.util.xrn import hrn_to_urn, get_leaf -from sfa.util.plxrn import slicename_to_hrn, hostname_to_hrn, hrn_to_pl_slicename, hrn_to_pl_login_base +from sfa.util.sfatime import utcparse +from sfa.util.xrn import hrn_to_urn, get_leaf, urn_to_sliver_id +from sfa.util.cache import Cache # one would think the driver should not need to mess with the SFA db, but.. from sfa.storage.table import SfaTable +# used to be used in get_ticket +#from sfa.trust.sfaticket import SfaTicket + +from sfa.rspecs.version_manager import VersionManager +from sfa.rspecs.rspec import RSpec + # the driver interface, mostly provides default behaviours from sfa.managers.driver import Driver from sfa.plc.plshell import PlShell +import sfa.plc.peers as peers +from sfa.plc.plaggregate import PlAggregate +from sfa.plc.plslices import PlSlices +from sfa.util.plxrn import slicename_to_hrn, hostname_to_hrn, hrn_to_pl_slicename, hrn_to_pl_login_base + def list_to_dict(recs, key): """ @@ -32,10 +48,21 @@ def list_to_dict(recs, key): # class PlDriver (Driver, PlShell): + # the cache instance is a class member so it survives across incoming requests + cache = None + def __init__ (self, config): PlShell.__init__ (self, config) + Driver.__init__ (self, config) + self.cache=None + if config.SFA_AGGREGATE_CACHING: + if PlDriver.cache is None: + PlDriver.cache = Cache() + self.cache = PlDriver.cache - self.hrn = config.SFA_INTERFACE_HRN + ######################################## + ########## registry oriented + ######################################## ########## disabled users def is_enabled (self, record): @@ -94,9 +121,9 @@ class PlDriver (Driver, PlShell): elif type == 'node': login_base = hrn_to_pl_login_base(sfa_record['authority']) - nodes = api.driver.GetNodes([pl_record['hostname']]) + nodes = self.GetNodes([pl_record['hostname']]) if not nodes: - pointer = api.driver.AddNode(login_base, pl_record) + pointer = self.AddNode(login_base, pl_record) else: pointer = nodes[0]['node_id'] @@ -381,7 +408,6 @@ class PlDriver (Driver, PlShell): return records - # aggregates is basically api.aggregates def fill_record_sfa_info(self, records): def startswith(prefix, values): @@ -499,14 +525,306 @@ class PlDriver (Driver, PlShell): current_target_ids = subject['person_ids'] add_target_ids = list ( set (target_ids).difference(current_target_ids)) del_target_ids = list ( set (current_target_ids).difference(target_ids)) - logger.info ("subject_id = %s (type=%s)"%(subject_id,type(subject_id))) + logger.debug ("subject_id = %s (type=%s)"%(subject_id,type(subject_id))) for target_id in add_target_ids: self.AddPersonToSlice (target_id,subject_id) - logger.info ("add_target_id = %s (type=%s)"%(target_id,type(target_id))) + logger.debug ("add_target_id = %s (type=%s)"%(target_id,type(target_id))) for target_id in del_target_ids: - logger.info ("del_target_id = %s (type=%s)"%(target_id,type(target_id))) + logger.debug ("del_target_id = %s (type=%s)"%(target_id,type(target_id))) self.DeletePersonFromSlice (target_id, subject_id) else: logger.info('unexpected relation to maintain, %s -> %s'%(subject_type,target_type)) + ######################################## + ########## aggregate oriented + ######################################## + + def testbed_name (self): return "myplc" + + # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory + def aggregate_version (self): + version_manager = VersionManager() + ad_rspec_versions = [] + request_rspec_versions = [] + for rspec_version in version_manager.versions: + if rspec_version.content_type in ['*', 'ad']: + ad_rspec_versions.append(rspec_version.to_dict()) + if rspec_version.content_type in ['*', 'request']: + request_rspec_versions.append(rspec_version.to_dict()) + return { + 'testbed':self.testbed_name(), + 'geni_request_rspec_versions': request_rspec_versions, + 'geni_ad_rspec_versions': ad_rspec_versions, + } + + def list_slices (self, creds, options): + # look in cache first + if self.cache: + slices = self.cache.get('slices') + if slices: + logger.debug("PlDriver.list_slices returns from cache") + return slices + + # get data from db + slices = self.GetSlices({'peer_id': None}, ['name']) + slice_hrns = [slicename_to_hrn(self.hrn, slice['name']) for slice in slices] + slice_urns = [hrn_to_urn(slice_hrn, 'slice') for slice_hrn in slice_hrns] + + # cache the result + if self.cache: + logger.debug ("PlDriver.list_slices stores value in cache") + self.cache.add('slices', slice_urns) + + return slice_urns + + # first 2 args are None in case of resource discovery + def list_resources (self, slice_urn, slice_hrn, creds, options): + cached_requested = options.get('cached', True) + + version_manager = VersionManager() + # get the rspec's return format from options + rspec_version = version_manager.get_version(options.get('geni_rspec_version')) + version_string = "rspec_%s" % (rspec_version) + + #panos adding the info option to the caching key (can be improved) + if options.get('info'): + version_string = version_string + "_"+options.get('info', 'default') + + # look in cache first + if cached_requested and self.cache and not slice_hrn: + rspec = self.cache.get(version_string) + if rspec: + logger.debug("PlDriver.ListResources: returning cached advertisement") + return rspec + + #panos: passing user-defined options + #print "manager options = ",options + aggregate = PlAggregate(self) + rspec = aggregate.get_rspec(slice_xrn=slice_urn, version=rspec_version, + options=options) + + # cache the result + if self.cache and not slice_hrn: + logger.debug("PlDriver.ListResources: stores advertisement in cache") + self.cache.add(version_string, rspec) + + return rspec + + def sliver_status (self, slice_urn, slice_hrn): + # find out where this slice is currently running + slicename = hrn_to_pl_slicename(slice_hrn) + + slices = self.GetSlices([slicename], ['slice_id', 'node_ids','person_ids','name','expires']) + if len(slices) == 0: + raise SliverDoesNotExist("%s (used %s as slicename internally)" % (slice_hrn, slicename)) + slice = slices[0] + + # report about the local nodes only + nodes = self.GetNodes({'node_id':slice['node_ids'],'peer_id':None}, + ['node_id', 'hostname', 'site_id', 'boot_state', 'last_contact']) + site_ids = [node['site_id'] for node in nodes] + + result = {} + top_level_status = 'unknown' + if nodes: + top_level_status = 'ready' + result['geni_urn'] = slice_urn + result['pl_login'] = slice['name'] + result['pl_expires'] = datetime.datetime.fromtimestamp(slice['expires']).ctime() + + resources = [] + for node in nodes: + res = {} + res['pl_hostname'] = node['hostname'] + res['pl_boot_state'] = node['boot_state'] + res['pl_last_contact'] = node['last_contact'] + if node['last_contact'] is not None: + res['pl_last_contact'] = datetime.datetime.fromtimestamp(node['last_contact']).ctime() + sliver_id = urn_to_sliver_id(slice_urn, slice['slice_id'], node['node_id']) + res['geni_urn'] = sliver_id + if node['boot_state'] == 'boot': + res['geni_status'] = 'ready' + else: + res['geni_status'] = 'failed' + top_level_status = 'failed' + + res['geni_error'] = '' + + resources.append(res) + + result['geni_status'] = top_level_status + result['geni_resources'] = resources + return result + + def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, users, options): + + aggregate = PlAggregate(self) + slices = PlSlices(self) + peer = slices.get_peer(slice_hrn) + sfa_peer = slices.get_sfa_peer(slice_hrn) + slice_record=None + if users: + slice_record = users[0].get('slice_record', {}) + + # parse rspec + rspec = RSpec(rspec_string) + requested_attributes = rspec.version.get_slice_attributes() + + # ensure site record exists + site = slices.verify_site(slice_hrn, slice_record, peer, sfa_peer, options=options) + # ensure slice record exists + slice = slices.verify_slice(slice_hrn, slice_record, peer, sfa_peer, options=options) + # ensure person records exists + persons = slices.verify_persons(slice_hrn, slice, users, peer, sfa_peer, options=options) + # ensure slice attributes exists + slices.verify_slice_attributes(slice, requested_attributes, options=options) + + # add/remove slice from nodes + requested_slivers = [node.get('component_name') for node in rspec.version.get_nodes_with_slivers()] + nodes = slices.verify_slice_nodes(slice, requested_slivers, peer) + + # add/remove links links + slices.verify_slice_links(slice, rspec.version.get_link_requests(), nodes) + + # handle MyPLC peer association. + # only used by plc and ple. + slices.handle_peer(site, slice, persons, peer) + + return aggregate.get_rspec(slice_xrn=slice_urn, version=rspec.version) + + def delete_sliver (self, slice_urn, slice_hrn, creds, options): + slicename = hrn_to_pl_slicename(slice_hrn) + slices = self.GetSlices({'name': slicename}) + if not slices: + return 1 + slice = slices[0] + + # determine if this is a peer slice + # xxx I wonder if this would not need to use PlSlices.get_peer instead + # in which case plc.peers could be deprecated as this here + # is the only/last call to this last method in plc.peers + peer = peers.get_peer(self, slice_hrn) + try: + if peer: + self.UnBindObjectFromPeer('slice', slice['slice_id'], peer) + self.DeleteSliceFromNodes(slicename, slice['node_ids']) + finally: + if peer: + self.BindObjectToPeer('slice', slice['slice_id'], peer, slice['peer_slice_id']) + return 1 + + def renew_sliver (self, slice_urn, slice_hrn, creds, expiration_time, options): + slicename = hrn_to_pl_slicename(slice_hrn) + slices = self.GetSlices({'name': slicename}, ['slice_id']) + if not slices: + raise RecordNotFound(slice_hrn) + slice = slices[0] + requested_time = utcparse(expiration_time) + record = {'expires': int(time.mktime(requested_time.timetuple()))} + try: + self.UpdateSlice(slice['slice_id'], record) + return True + except: + return False + + # remove the 'enabled' tag + def start_slice (self, slice_urn, slice_hrn, creds): + slicename = hrn_to_pl_slicename(slice_hrn) + slices = self.GetSlices({'name': slicename}, ['slice_id']) + if not slices: + raise RecordNotFound(slice_hrn) + slice_id = slices[0]['slice_id'] + slice_tags = self.GetSliceTags({'slice_id': slice_id, 'tagname': 'enabled'}, ['slice_tag_id']) + # just remove the tag if it exists + if slice_tags: + self.DeleteSliceTag(slice_tags[0]['slice_tag_id']) + return 1 + + # set the 'enabled' tag to 0 + def stop_slice (self, slice_urn, slice_hrn, creds): + slicename = hrn_to_pl_slicename(slice_hrn) + slices = self.GetSlices({'name': slicename}, ['slice_id']) + if not slices: + raise RecordNotFound(slice_hrn) + slice_id = slices[0]['slice_id'] + slice_tags = self.GetSliceTags({'slice_id': slice_id, 'tagname': 'enabled'}) + if not slice_tags: + self.AddSliceTag(slice_id, 'enabled', '0') + elif slice_tags[0]['value'] != "0": + tag_id = slice_tags[0]['slice_tag_id'] + self.UpdateSliceTag(tag_id, '0') + return 1 + + def reset_slice (self, slice_urn, slice_hrn, creds): + raise SfaNotImplemented ("reset_slice not available at this interface") + + # xxx this code is quite old and has not run for ages + # it is obviously totally broken and needs a rewrite + def get_ticket (self, slice_urn, slice_hrn, creds, rspec_string, options): + raise SfaNotImplemented,"PlDriver.get_ticket needs a rewrite" +# please keep this code for future reference +# slices = PlSlices(self) +# peer = slices.get_peer(slice_hrn) +# sfa_peer = slices.get_sfa_peer(slice_hrn) +# +# # get the slice record +# credential = api.getCredential() +# interface = api.registries[api.hrn] +# registry = api.server_proxy(interface, credential) +# records = registry.Resolve(xrn, credential) +# +# # make sure we get a local slice record +# record = None +# for tmp_record in records: +# if tmp_record['type'] == 'slice' and \ +# not tmp_record['peer_authority']: +# #Error (E0602, GetTicket): Undefined variable 'SliceRecord' +# slice_record = SliceRecord(dict=tmp_record) +# if not record: +# raise RecordNotFound(slice_hrn) +# +# # similar to CreateSliver, we must verify that the required records exist +# # at this aggregate before we can issue a ticket +# # parse rspec +# rspec = RSpec(rspec_string) +# requested_attributes = rspec.version.get_slice_attributes() +# +# # ensure site record exists +# site = slices.verify_site(slice_hrn, slice_record, peer, sfa_peer) +# # ensure slice record exists +# slice = slices.verify_slice(slice_hrn, slice_record, peer, sfa_peer) +# # ensure person records exists +# # xxx users is undefined in this context +# persons = slices.verify_persons(slice_hrn, slice, users, peer, sfa_peer) +# # ensure slice attributes exists +# slices.verify_slice_attributes(slice, requested_attributes) +# +# # get sliver info +# slivers = slices.get_slivers(slice_hrn) +# +# if not slivers: +# raise SliverDoesNotExist(slice_hrn) +# +# # get initscripts +# initscripts = [] +# data = { +# 'timestamp': int(time.time()), +# 'initscripts': initscripts, +# 'slivers': slivers +# } +# +# # create the ticket +# object_gid = record.get_gid_object() +# new_ticket = SfaTicket(subject = object_gid.get_subject()) +# new_ticket.set_gid_caller(api.auth.client_gid) +# new_ticket.set_gid_object(object_gid) +# new_ticket.set_issuer(key=api.key, subject=self.hrn) +# new_ticket.set_pubkey(object_gid.get_pubkey()) +# new_ticket.set_attributes(data) +# new_ticket.set_rspec(rspec) +# #new_ticket.set_parent(api.auth.hierarchy.get_auth_ticket(auth_hrn)) +# new_ticket.encode() +# new_ticket.sign() +# +# return new_ticket.save_to_string(save_parents=True) diff --git a/sfa/plc/plshell.py b/sfa/plc/plshell.py index eaa772a9..eea33c0d 100644 --- a/sfa/plc/plshell.py +++ b/sfa/plc/plshell.py @@ -37,10 +37,16 @@ class PlShell: hostname=urlparse(url).hostname is_local=False if hostname == 'localhost': is_local=True - # otherwise compare IP addresses - url_ip=socket.gethostbyname(hostname) - local_ip=socket.gethostbyname(socket.gethostname()) - if url_ip==local_ip: is_local=True + # otherwise compare IP addresses; + # this might fail for any number of reasons, so let's harden that + try: + # xxx todo this seems to result in a DNS request for each incoming request to the AM + # should be cached or improved + url_ip=socket.gethostbyname(hostname) + local_ip=socket.gethostbyname(socket.gethostname()) + if url_ip==local_ip: is_local=True + except: + pass if is_local: try: @@ -65,7 +71,7 @@ class PlShell: 'Username': config.SFA_PLC_USER, 'AuthString': config.SFA_PLC_PASSWORD, } - self.proxy = xmlrpclib.Server(url, verbose = 0, allow_none = True) + self.proxy = xmlrpclib.Server(url, verbose = False, allow_none = True) def __getattr__(self, name): def func(*args, **kwds): diff --git a/sfa/plc/slices.py b/sfa/plc/plslices.py similarity index 61% rename from sfa/plc/slices.py rename to sfa/plc/plslices.py index dc317935..ae9a0796 100644 --- a/sfa/plc/slices.py +++ b/sfa/plc/plslices.py @@ -1,28 +1,25 @@ from types import StringTypes from collections import defaultdict import sys -from sfa.util.xrn import get_leaf, get_authority, urn_to_hrn -from sfa.util.plxrn import hrn_to_pl_slicename -from sfa.util.policy import Policy + +from sfa.util.sfalogging import logger +from sfa.util.xrn import Xrn, get_leaf, get_authority, urn_to_hrn +#from sfa.util.policy import Policy +from sfa.util.xrn import Xrn + from sfa.rspecs.rspec import RSpec + from sfa.plc.vlink import VLink -from sfa.util.xrn import Xrn +from sfa.util.plxrn import hrn_to_pl_slicename MAXINT = 2L**31-1 -class Slices: +class PlSlices: rspec_to_slice_tag = {'max_rate':'net_max_rate'} - def __init__(self, api, ttl = .5, origin_hrn=None): - self.api = api - #filepath = path + os.sep + filename - self.policy = Policy(self.api) - self.origin_hrn = origin_hrn - self.registry = api.registries[api.hrn] - self.credential = api.getCredential() - self.nodes = [] - self.persons = [] + def __init__(self, driver): + self.driver = driver def get_slivers(self, xrn, node=None): hrn, type = urn_to_hrn(xrn) @@ -30,11 +27,11 @@ class Slices: slice_name = hrn_to_pl_slicename(hrn) # XX Should we just call PLCAPI.GetSliceTicket(slice_name) instead # of doing all of this? - #return self.api.driver.GetSliceTicket(self.auth, slice_name) + #return self.driver.GetSliceTicket(self.auth, slice_name) # from PLCAPI.GetSlivers.get_slivers() slice_fields = ['slice_id', 'name', 'instantiation', 'expires', 'person_ids', 'slice_tag_ids'] - slices = self.api.driver.GetSlices(slice_name, slice_fields) + slices = self.driver.GetSlices(slice_name, slice_fields) # Build up list of users and slice attributes person_ids = set() all_slice_tag_ids = set() @@ -44,7 +41,7 @@ class Slices: person_ids = list(person_ids) all_slice_tag_ids = list(all_slice_tag_ids) # Get user information - all_persons_list = self.api.driver.GetPersons({'person_id':person_ids,'enabled':True}, ['person_id', 'enabled', 'key_ids']) + all_persons_list = self.driver.GetPersons({'person_id':person_ids,'enabled':True}, ['person_id', 'enabled', 'key_ids']) all_persons = {} for person in all_persons_list: all_persons[person['person_id']] = person @@ -55,12 +52,12 @@ class Slices: key_ids.update(person['key_ids']) key_ids = list(key_ids) # Get user account keys - all_keys_list = self.api.driver.GetKeys(key_ids, ['key_id', 'key', 'key_type']) + all_keys_list = self.driver.GetKeys(key_ids, ['key_id', 'key', 'key_type']) all_keys = {} for key in all_keys_list: all_keys[key['key_id']] = key # Get slice attributes - all_slice_tags_list = self.api.driver.GetSliceTags(all_slice_tag_ids) + all_slice_tags_list = self.driver.GetSliceTags(all_slice_tag_ids) all_slice_tags = {} for slice_tag in all_slice_tags_list: all_slice_tags[slice_tag['slice_tag_id']] = slice_tag @@ -135,19 +132,19 @@ class Slices: # slice belongs to out local plc or a myplc peer. We will assume it # is a local site, unless we find out otherwise peer = None - print>>sys.stderr, " \r\n \r\n \t slices.py get_peer slice_authority " + print>>sys.stderr, " \r\n \r\n \tplslices.py get_peer slice_authority " # get this slice's authority (site) slice_authority = get_authority(hrn) # get this site's authority (sfa root authority or sub authority) site_authority = get_authority(slice_authority).lower() - print>>sys.stderr, " \r\n \r\n \t slices.py get_peer slice_authority %s site_authority %s" %(slice_authority,site_authority) + print>>sys.stderr, " \r\n \r\n \tplslices.py get_peer slice_authority %s site_authority %s" %(slice_authority,site_authority) # check if we are already peered with this site_authority, if so - #peers = self.api.driver.GetPeers({}, ['peer_id', 'peername', 'shortname', 'hrn_root']) - #for peer_record in peers: - #names = [name.lower() for name in peer_record.values() if isinstance(name, StringTypes)] - #if site_authority in names: - #peer = peer_record + peers = self.driver.GetPeers({}, ['peer_id', 'peername', 'shortname', 'hrn_root']) + for peer_record in peers: + names = [name.lower() for name in peer_record.values() if isinstance(name, StringTypes)] + if site_authority in names: + peer = peer_record return peer @@ -159,14 +156,14 @@ class Slices: slice_authority = get_authority(hrn) site_authority = get_authority(slice_authority) - if site_authority != self.api.hrn: + if site_authority != self.driver.hrn: sfa_peer = site_authority return sfa_peer def verify_slice_nodes(self, slice, requested_slivers, peer): - nodes = self.api.driver.GetNodes(slice['node_ids'], ['hostname']) + nodes = self.driver.GetNodes(slice['node_ids'], ['node_id', 'hostname', 'interface_ids']) current_slivers = [node['hostname'] for node in nodes] # remove nodes not in rspec @@ -177,16 +174,17 @@ class Slices: try: if peer: - self.api.driver.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname']) - self.api.driver.AddSliceToNodes(slice['name'], added_nodes) - self.api.driver.DeleteSliceFromNodes(slice['name'], deleted_nodes) + self.driver.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname']) + self.driver.AddSliceToNodes(slice['name'], added_nodes) + self.driver.DeleteSliceFromNodes(slice['name'], deleted_nodes) except: - self.api.logger.log_exc('Failed to add/remove slice from nodes') + logger.log_exc('Failed to add/remove slice from nodes') + return nodes def free_egre_key(self): used = set() - for tag in self.api.driver.GetSliceTags({'tagname': 'egre_key'}): + for tag in self.driver.GetSliceTags({'tagname': 'egre_key'}): used.add(int(tag['value'])) for i in range(1, 256): @@ -198,10 +196,22 @@ class Slices: return str(key) - def verify_slice_links(self, slice, links, aggregate): + def verify_slice_links(self, slice, requested_links, nodes): # nodes is undefined here - if not links: + if not requested_links: return + + # build dict of nodes + nodes_dict = {} + interface_ids = [] + for node in nodes: + nodes_dict[node['node_id']] = node + interface_ids.extend(node['interface_ids']) + # build dict of interfaces + interfaces = self.driver.GetInterfaces(interface_ids) + interfaces_dict = {} + for interface in interfaces: + interfaces_dict[interface['interface_id']] = interface slice_tags = [] @@ -215,22 +225,22 @@ class Slices: # need to update the attribute string? slice_tags.append({'name': 'capabilities', 'value': 'CAP_NET_ADMIN'}) - for link in links: + for link in requested_links: # get the ip address of the first node in the link ifname1 = Xrn(link['interface1']['component_id']).get_leaf() - (node, device) = ifname1.split(':') - node_id = int(node.replace('node', '')) - node = aggregate.nodes[node_id] - if1 = aggregate.interfaces[node['interface_ids'][0]] + (node_raw, device) = ifname1.split(':') + node_id = int(node_raw.replace('node', '')) + node = nodes_dict[node_id] + if1 = interfaces_dict[node['interface_ids'][0]] ipaddr = if1['ip'] topo_rspec = VLink.get_topo_rspec(link, ipaddr) # set topo_rspec tag slice_tags.append({'name': 'topo_rspec', 'value': str([topo_rspec]), 'node_id': node_id}) # set vini_topo tag slice_tags.append({'name': 'vini_topo', 'value': 'manual', 'node_id': node_id}) - #self.api.driver.AddSliceTag(slice['name'], 'topo_rspec', str([topo_rspec]), node_id) + #self.driver.AddSliceTag(slice['name'], 'topo_rspec', str([topo_rspec]), node_id) - self.verify_slice_attributes(slice, slice_tags, append=True, admin=True) + self.verify_slice_attributes(slice, slice_tags, {'append': True}, admin=True) @@ -239,45 +249,45 @@ class Slices: # bind site try: if site: - self.api.driver.BindObjectToPeer('site', site['site_id'], peer['shortname'], slice['site_id']) + self.driver.BindObjectToPeer('site', site['site_id'], peer['shortname'], slice['site_id']) except Exception,e: - self.api.driver.DeleteSite(site['site_id']) + self.driver.DeleteSite(site['site_id']) raise e # bind slice try: if slice: - self.api.driver.BindObjectToPeer('slice', slice['slice_id'], peer['shortname'], slice['slice_id']) + self.driver.BindObjectToPeer('slice', slice['slice_id'], peer['shortname'], slice['slice_id']) except Exception,e: - self.api.driver.DeleteSlice(slice['slice_id']) + self.driver.DeleteSlice(slice['slice_id']) raise e # bind persons for person in persons: try: - self.api.driver.BindObjectToPeer('person', + self.driver.BindObjectToPeer('person', person['person_id'], peer['shortname'], person['peer_person_id']) for (key, remote_key_id) in zip(person['keys'], person['key_ids']): try: - self.api.driver.BindObjectToPeer( 'key', key['key_id'], peer['shortname'], remote_key_id) + self.driver.BindObjectToPeer( 'key', key['key_id'], peer['shortname'], remote_key_id) except: - self.api.driver.DeleteKey(key['key_id']) - self.api.logger("failed to bind key: %s to peer: %s " % (key['key_id'], peer['shortname'])) + self.driver.DeleteKey(key['key_id']) + logger("failed to bind key: %s to peer: %s " % (key['key_id'], peer['shortname'])) except Exception,e: - self.api.driver.DeletePerson(person['person_id']) + self.driver.DeletePerson(person['person_id']) raise e return slice - def verify_site(self, slice_xrn, slice_record={}, peer=None, sfa_peer=None): + def verify_site(self, slice_xrn, slice_record={}, peer=None, sfa_peer=None, options={}): (slice_hrn, type) = urn_to_hrn(slice_xrn) site_hrn = get_authority(slice_hrn) # login base can't be longer than 20 characters slicename = hrn_to_pl_slicename(slice_hrn) authority_name = slicename.split('_')[0] login_base = authority_name[:20] - sites = self.api.driver.GetSites(login_base) + sites = self.driver.GetSites(login_base) if not sites: # create new site record site = {'name': 'geni.%s' % authority_name, @@ -289,97 +299,102 @@ class Slices: 'peer_site_id': None} if peer: site['peer_site_id'] = slice_record.get('site_id', None) - site['site_id'] = self.api.driver.AddSite(site) + site['site_id'] = self.driver.AddSite(site) # exempt federated sites from monitor policies - self.api.driver.AddSiteTag(site['site_id'], 'exempt_site_until', "20200101") + self.driver.AddSiteTag(site['site_id'], 'exempt_site_until', "20200101") - # is this still necessary? - # add record to the local registry - if sfa_peer and slice_record: - peer_dict = {'type': 'authority', 'hrn': site_hrn, \ - 'peer_authority': sfa_peer, 'pointer': site['site_id']} - self.registry.register_peer_object(self.credential, peer_dict) +# # is this still necessary? +# # add record to the local registry +# if sfa_peer and slice_record: +# peer_dict = {'type': 'authority', 'hrn': site_hrn, \ +# 'peer_authority': sfa_peer, 'pointer': site['site_id']} +# self.registry.register_peer_object(self.credential, peer_dict) else: site = sites[0] if peer: # unbind from peer so we can modify if necessary. Will bind back later - self.api.driver.UnBindObjectFromPeer('site', site['site_id'], peer['shortname']) + self.driver.UnBindObjectFromPeer('site', site['site_id'], peer['shortname']) return site - def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer): - #slicename = hrn_to_pl_slicename(slice_hrn) - parts = hrn_to_pl_slicename(slice_hrn).split("_") + def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer, options={}): + slicename = hrn_to_pl_slicename(slice_hrn) + parts = slicename.split("_") login_base = parts[0] - slicename = slice_hrn - slices = self.api.driver.GetSlices([slicename]) - print>>sys.stderr, " \r\n \r\rn Slices.py verify_slice slicename %s slices %s slice_record %s"%(slicename ,slices, slice_record) + slices = self.driver.GetSlices([slicename]) if not slices: slice = {'name': slicename, 'url': slice_record.get('url', slice_hrn), 'description': slice_record.get('description', slice_hrn)} # add the slice - slice['slice_id'] = self.api.driver.AddSlice(slice) + slice['slice_id'] = self.driver.AddSlice(slice) slice['node_ids'] = [] slice['person_ids'] = [] if peer: slice['peer_slice_id'] = slice_record.get('slice_id', None) # mark this slice as an sfa peer record - if sfa_peer: - peer_dict = {'type': 'slice', 'hrn': slice_hrn, - 'peer_authority': sfa_peer, 'pointer': slice['slice_id']} - self.registry.register_peer_object(self.credential, peer_dict) +# if sfa_peer: +# peer_dict = {'type': 'slice', 'hrn': slice_hrn, +# 'peer_authority': sfa_peer, 'pointer': slice['slice_id']} +# self.registry.register_peer_object(self.credential, peer_dict) else: slice = slices[0] if peer: slice['peer_slice_id'] = slice_record.get('slice_id', None) # unbind from peer so we can modify if necessary. Will bind back later - self.api.driver.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname']) + self.driver.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname']) #Update existing record (e.g. expires field) it with the latest info. - #if slice_record and slice['expires'] != slice_record['expires']: - #self.api.driver.UpdateSlice( slice['slice_id'], {'expires' : slice_record['expires']}) + if slice_record and slice['expires'] != slice_record['expires']: + self.driver.UpdateSlice( slice['slice_id'], {'expires' : slice_record['expires']}) return slice #def get_existing_persons(self, users): - def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, append=True): + def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, options={}): users_by_email = {} users_by_site = defaultdict(list) - users_dict = {} for user in users: - if 'append' in user and user['append'] == False: - append = False + hrn, type = urn_to_hrn(user['urn']) + username = get_leaf(hrn) + login_base = get_leaf(get_authority(user['urn'])) + user['username'] = username + user['site'] = login_base + if 'email' in user: users_by_email[user['email']] = user users_dict[user['email']] = user - elif 'urn' in user: - hrn, type = urn_to_hrn(user['urn']) - username = get_leaf(hrn) - login_base = get_leaf(get_authority(user['urn'])) - user['username'] = username - users_by_site[login_base].append(user) + else: + users_by_site[user['site']].append(user) + # start building a list of existing users existing_user_ids = [] + existing_user_ids_filter = [] if users_by_email: + existing_user_ids_filter.extend(users_by_email.keys()) + if users_by_site: + for login_base in users_by_site: + users = users_by_site[login_base] + for user in users: + existing_user_ids_filter.append(user['username']+'@geni.net') + if existing_user_ids_filter: # get existing users by email - existing_users = self.api.driver.GetPersons({'email': users_by_email.keys()}, + existing_users = self.driver.GetPersons({'email': existing_user_ids_filter}, ['person_id', 'key_ids', 'email']) existing_user_ids.extend([user['email'] for user in existing_users]) - + if users_by_site: # get a list of user sites (based on requeste user urns - site_list = self.api.driver.GetSites(users_by_site.keys(), \ + site_list = self.driver.GetSites(users_by_site.keys(), \ ['site_id', 'login_base', 'person_ids']) + # get all existing users at these sites sites = {} site_user_ids = [] - - # get all existing users at these sites for site in site_list: sites[site['site_id']] = site site_user_ids.extend(site['person_ids']) - existing_site_persons_list = self.api.driver.GetPersons(site_user_ids, + existing_site_persons_list = self.driver.GetPersons(site_user_ids, ['person_id', 'key_ids', 'email', 'site_ids']) # all requested users are either existing users or new (added) users @@ -389,13 +404,14 @@ class Slices: user_found = False for existing_user in existing_site_persons_list: for site_id in existing_user['site_ids']: - site = sites[site_id] - if login_base == site['login_base'] and \ - existing_user['email'].startswith(requested_user['username']): - existing_user_ids.append(existing_user['email']) - users_dict[existing_user['email']] = requested_user - user_found = True - break + if site_id in sites: + site = sites[site_id] + if login_base == site['login_base'] and \ + existing_user['email'].startswith(requested_user['username']+'@'): + existing_user_ids.append(existing_user['email']) + users_dict[existing_user['email']] = requested_user + user_found = True + break if user_found: break @@ -403,12 +419,11 @@ class Slices: fake_email = requested_user['username'] + '@geni.net' users_dict[fake_email] = requested_user - # requested slice users requested_user_ids = users_dict.keys() # existing slice users existing_slice_users_filter = {'person_id': slice_record.get('person_ids', [])} - existing_slice_users = self.api.driver.GetPersons(existing_slice_users_filter, + existing_slice_users = self.driver.GetPersons(existing_slice_users_filter, ['person_id', 'key_ids', 'email']) existing_slice_user_ids = [user['email'] for user in existing_slice_users] @@ -419,13 +434,15 @@ class Slices: updated_user_ids = set(existing_slice_user_ids).intersection(requested_user_ids) # Remove stale users (only if we are not appending). + # Append by default. + append = options.get('append', True) if append == False: for removed_user_id in removed_user_ids: - self.api.driver.DeletePersonFromSlice(removed_user_id, slice_record['name']) + self.driver.DeletePersonFromSlice(removed_user_id, slice_record['name']) # update_existing users updated_users_list = [user for user in existing_slice_users if user['email'] in \ updated_user_ids] - self.verify_keys(existing_slice_users, updated_users_list, peer, append) + self.verify_keys(existing_slice_users, updated_users_list, peer, options) added_persons = [] # add new users @@ -440,43 +457,43 @@ class Slices: 'keys': [], 'key_ids': added_user.get('key_ids', []), } - person['person_id'] = self.api.driver.AddPerson(person) + person['person_id'] = self.driver.AddPerson(person) if peer: person['peer_person_id'] = added_user['person_id'] added_persons.append(person) # enable the account - self.api.driver.UpdatePerson(person['person_id'], {'enabled': True}) + self.driver.UpdatePerson(person['person_id'], {'enabled': True}) # add person to site - self.api.driver.AddPersonToSite(added_user_id, login_base) + self.driver.AddPersonToSite(added_user_id, added_user['site']) for key_string in added_user.get('keys', []): key = {'key':key_string, 'key_type':'ssh'} - key['key_id'] = self.api.driver.AddPersonKey(person['person_id'], key) + key['key_id'] = self.driver.AddPersonKey(person['person_id'], key) person['keys'].append(key) # add the registry record - if sfa_peer: - peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': sfa_peer, \ - 'pointer': person['person_id']} - self.registry.register_peer_object(self.credential, peer_dict) +# if sfa_peer: +# peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': sfa_peer, \ +# 'pointer': person['person_id']} +# self.registry.register_peer_object(self.credential, peer_dict) for added_slice_user_id in added_slice_user_ids.union(added_user_ids): # add person to the slice - self.api.driver.AddPersonToSlice(added_slice_user_id, slice_record['name']) + self.driver.AddPersonToSlice(added_slice_user_id, slice_record['name']) # if this is a peer record then it should already be bound to a peer. # no need to return worry about it getting bound later return added_persons - def verify_keys(self, persons, users, peer, append=True): + def verify_keys(self, persons, users, peer, options={}): # existing keys key_ids = [] for person in persons: key_ids.extend(person['key_ids']) - keylist = self.api.driver.GetKeys(key_ids, ['key_id', 'key']) + keylist = self.driver.GetKeys(key_ids, ['key_id', 'key']) keydict = {} for key in keylist: keydict[key['key']] = key['key_id'] @@ -498,43 +515,45 @@ class Slices: try: if peer: person = persondict[user['email']] - self.api.driver.UnBindObjectFromPeer('person', person['person_id'], peer['shortname']) - key['key_id'] = self.api.driver.AddPersonKey(user['email'], key) + self.driver.UnBindObjectFromPeer('person', person['person_id'], peer['shortname']) + key['key_id'] = self.driver.AddPersonKey(user['email'], key) if peer: key_index = user_keys.index(key['key']) remote_key_id = user['key_ids'][key_index] - self.api.driver.BindObjectToPeer('key', key['key_id'], peer['shortname'], remote_key_id) + self.driver.BindObjectToPeer('key', key['key_id'], peer['shortname'], remote_key_id) finally: if peer: - self.api.driver.BindObjectToPeer('person', person['person_id'], peer['shortname'], user['person_id']) + self.driver.BindObjectToPeer('person', person['person_id'], peer['shortname'], user['person_id']) # remove old keys (only if we are not appending) + append = options.get('append', True) if append == False: removed_keys = set(existing_keys).difference(requested_keys) for existing_key_id in keydict: if keydict[existing_key_id] in removed_keys: try: if peer: - self.api.driver.UnBindObjectFromPeer('key', existing_key_id, peer['shortname']) - self.api.driver.DeleteKey(existing_key_id) + self.driver.UnBindObjectFromPeer('key', existing_key_id, peer['shortname']) + self.driver.DeleteKey(existing_key_id) except: pass - def verify_slice_attributes(self, slice, requested_slice_attributes, append=False, admin=False): + def verify_slice_attributes(self, slice, requested_slice_attributes, options={}, admin=False): + append = options.get('append', True) # get list of attributes users ar able to manage filter = {'category': '*slice*'} if not admin: filter['|roles'] = ['user'] - slice_attributes = self.api.driver.GetTagTypes(filter) + slice_attributes = self.driver.GetTagTypes(filter) valid_slice_attribute_names = [attribute['tagname'] for attribute in slice_attributes] # get sliver attributes added_slice_attributes = [] removed_slice_attributes = [] ignored_slice_attribute_names = [] - existing_slice_attributes = self.api.driver.GetSliceTags({'slice_id': slice['slice_id']}) - + existing_slice_attributes = self.driver.GetSliceTags({'slice_id': slice['slice_id']}) + # get attributes that should be removed for slice_tag in existing_slice_attributes: if slice_tag['tagname'] in ignored_slice_attribute_names: @@ -571,95 +590,16 @@ class Slices: # remove stale attributes for attribute in removed_slice_attributes: try: - self.api.driver.DeleteSliceTag(attribute['slice_tag_id']) + self.driver.DeleteSliceTag(attribute['slice_tag_id']) except Exception, e: - self.api.logger.warn('Failed to remove sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\ + logger.warn('Failed to remove sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\ % (name, value, node_id, str(e))) # add requested_attributes for attribute in added_slice_attributes: try: - self.api.driver.AddSliceTag(slice['name'], attribute['name'], attribute['value'], attribute.get('node_id', None)) + self.driver.AddSliceTag(slice['name'], attribute['name'], attribute['value'], attribute.get('node_id', None)) except Exception, e: - self.api.logger.warn('Failed to add sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\ + logger.warn('Failed to add sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\ % (name, value, node_id, str(e))) - def create_slice_aggregate(self, xrn, rspec): - hrn, type = urn_to_hrn(xrn) - # Determine if this is a peer slice - peer = self.get_peer(hrn) - sfa_peer = self.get_sfa_peer(hrn) - - spec = RSpec(rspec) - # Get the slice record from sfa - slicename = hrn_to_pl_slicename(hrn) - slice = {} - slice_record = None - registry = self.api.registries[self.api.hrn] - credential = self.api.getCredential() - - site_id, remote_site_id = self.verify_site(registry, credential, hrn, peer, sfa_peer) - slice = self.verify_slice(registry, credential, hrn, site_id, remote_site_id, peer, sfa_peer) - - # find out where this slice is currently running - nodelist = self.api.driver.GetNodes(slice['node_ids'], ['hostname']) - hostnames = [node['hostname'] for node in nodelist] - - # get netspec details - nodespecs = spec.getDictsByTagName('NodeSpec') - - # dict in which to store slice attributes to set for the nodes - nodes = {} - for nodespec in nodespecs: - if isinstance(nodespec['name'], list): - for nodename in nodespec['name']: - nodes[nodename] = {} - for k in nodespec.keys(): - rspec_attribute_value = nodespec[k] - if (self.rspec_to_slice_tag.has_key(k)): - slice_tag_name = self.rspec_to_slice_tag[k] - nodes[nodename][slice_tag_name] = rspec_attribute_value - elif isinstance(nodespec['name'], StringTypes): - nodename = nodespec['name'] - nodes[nodename] = {} - for k in nodespec.keys(): - rspec_attribute_value = nodespec[k] - if (self.rspec_to_slice_tag.has_key(k)): - slice_tag_name = self.rspec_to_slice_tag[k] - nodes[nodename][slice_tag_name] = rspec_attribute_value - - for k in nodespec.keys(): - rspec_attribute_value = nodespec[k] - if (self.rspec_to_slice_tag.has_key(k)): - slice_tag_name = self.rspec_to_slice_tag[k] - nodes[nodename][slice_tag_name] = rspec_attribute_value - - node_names = nodes.keys() - # remove nodes not in rspec - deleted_nodes = list(set(hostnames).difference(node_names)) - # add nodes from rspec - added_nodes = list(set(node_names).difference(hostnames)) - - try: - if peer: - self.api.driver.UnBindObjectFromPeer('slice', slice['slice_id'], peer) - - self.api.driver.AddSliceToNodes(slicename, added_nodes) - - # Add recognized slice tags - for node_name in node_names: - node = nodes[node_name] - for slice_tag in node.keys(): - value = node[slice_tag] - if (isinstance(value, list)): - value = value[0] - - self.api.driver.AddSliceTag(slicename, slice_tag, value, node_name) - - self.api.driver.DeleteSliceFromNodes(slicename, deleted_nodes) - finally: - if peer: - self.api.driver.BindObjectToPeer('slice', slice['slice_id'], peer, slice['peer_slice_id']) - - return 1 - diff --git a/sfa/util/topology.py b/sfa/plc/topology.py similarity index 74% rename from sfa/util/topology.py rename to sfa/plc/topology.py index 79482f19..cd6036af 100644 --- a/sfa/util/topology.py +++ b/sfa/plc/topology.py @@ -15,16 +15,7 @@ class Topology(set): def __init__(self, config_file = "/etc/sfa/topology"): set.__init__(self) - self.config_file = None - self.config_path = None - self.load(config_file) - - def load(self, config_file): try: - - self.config_file = config_file - # path to configuration data - self.config_path = os.path.dirname(config_file) # load the links f = open(config_file, 'r') for line in f: diff --git a/sfa/rspecs/elements/versions/pgv2Link.py b/sfa/rspecs/elements/versions/pgv2Link.py index 44317be2..5e0a8282 100644 --- a/sfa/rspecs/elements/versions/pgv2Link.py +++ b/sfa/rspecs/elements/versions/pgv2Link.py @@ -49,11 +49,11 @@ class PGv2Link: # get capacity, latency and packet_loss from first property property_fields = ['capacity', 'latency', 'packet_loss'] property_elems = link_elem.xpath('./default:property | ./property') - if len(propery_elems) > 0: + if len(property_elems) > 0: prop = property_elems[0] for attrib in ['capacity', 'latency', 'packet_loss']: - if attrib in prop: - link[attrib] = prop[attrib] + if attrib in prop.attrib: + link[attrib] = prop.attrib[attrib] # get interfaces iface_elems = link_elem.xpath('./default:interface_ref | ./interface_ref') diff --git a/sfa/rspecs/elements/versions/pgv2SliverType.py b/sfa/rspecs/elements/versions/pgv2SliverType.py index 4271d211..94a9f639 100644 --- a/sfa/rspecs/elements/versions/pgv2SliverType.py +++ b/sfa/rspecs/elements/versions/pgv2SliverType.py @@ -10,7 +10,11 @@ class PGv2SliverType: if not isinstance(slivers, list): slivers = [slivers] for sliver in slivers: - sliver_elem = xml.add_instance('sliver_type', sliver, ['type', 'client_id']) + sliver_elem = xml.add_element('sliver_type') + if sliver.get('type'): + sliver_elem.set('name', sliver['type']) + if sliver.get('client_id'): + sliver_elem.set('client_id', sliver['client_id']) PGv2SliverType.add_sliver_attributes(sliver_elem, sliver.get('pl_tags', [])) @staticmethod @@ -32,6 +36,8 @@ class PGv2SliverType: sliver = Sliver(sliver_elem.attrib,sliver_elem) if 'component_id' in xml.attrib: sliver['component_id'] = xml.attrib['component_id'] + if 'name' in sliver_elem.attrib: + sliver['type'] = sliver_elem.attrib['name'] slivers.append(sliver) return slivers diff --git a/sfa/rspecs/elements/versions/sfav1Sliver.py b/sfa/rspecs/elements/versions/sfav1Sliver.py index 207a4a78..f0375336 100644 --- a/sfa/rspecs/elements/versions/sfav1Sliver.py +++ b/sfa/rspecs/elements/versions/sfav1Sliver.py @@ -1,4 +1,5 @@ from sfa.util.xrn import Xrn +from sfa.util.plxrn import PlXrn from sfa.util.xml import XmlElement from sfa.rspecs.elements.element import Element from sfa.rspecs.elements.sliver import Sliver @@ -19,9 +20,7 @@ class SFAv1Sliver: for tag in tags: SFAv1Sliver.add_sliver_attribute(sliver_elem, tag['tagname'], tag['value']) if sliver.get('sliver_id'): - sliver_id_leaf = Xrn(sliver.get('sliver_id')).get_leaf() - sliver_id_parts = sliver_id_leaf.split(':') - name = sliver_id_parts[0] + name = PlXrn(xrn=sliver.get('sliver_id')).pl_slicename() sliver_elem.set('name', name) @staticmethod @@ -35,7 +34,7 @@ class SFAv1Sliver: for elem in xml.iterchildren(): if elem.tag not in Sliver.fields: xml_element = XmlElement(elem, xml.namespaces) - instance = Element(xml_element) + instance = Element(fields=xml_element, element=elem) instance['name'] = elem.tag instance['value'] = elem.text attribs.append(instance) diff --git a/sfa/rspecs/extensions/ext/sfa/1/planetlab.rnc b/sfa/rspecs/extensions/ext/sfa/1/planetlab.rnc new file mode 100644 index 00000000..f9d7d17e --- /dev/null +++ b/sfa/rspecs/extensions/ext/sfa/1/planetlab.rnc @@ -0,0 +1,18 @@ +# +# Extensions for PlanetLab Resources +# Version 1 +# + +namespace planetlab = "http://www.planet-lab.org/resources/ext/planetlab/1" + +InitscriptSpec = element planetlab:initscript { + attribute name { text } +} + +AttributeSpec = element planetlab:attribute { + attribute name { text }, + attribute value { text } +} + +start = InitscriptSpec | AttributeSpec + diff --git a/sfa/rspecs/extensions/ext/sfa/1/planetlab.xsd b/sfa/rspecs/extensions/ext/sfa/1/planetlab.xsd new file mode 100644 index 00000000..f6b20821 --- /dev/null +++ b/sfa/rspecs/extensions/ext/sfa/1/planetlab.xsd @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + diff --git a/sfa/rspecs/resources/__init__.py b/sfa/rspecs/resources/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/sfa/rspecs/resources/ext/__init__.py b/sfa/rspecs/resources/ext/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/sfa/rspecs/resources/ext/planetlab.rnc b/sfa/rspecs/resources/ext/planetlab.rnc deleted file mode 100644 index f1ff971e..00000000 --- a/sfa/rspecs/resources/ext/planetlab.rnc +++ /dev/null @@ -1,13 +0,0 @@ -# -## Extension for the "initscript" type for RSpecV2 on PlanetLab -## Version 1 -## - -default namespace = "http://www.planet-lab.org/resources/ext/initscript/1" - -Node = element initscript { - attribute name { text } -} - -start = Node - diff --git a/sfa/rspecs/resources/ext/planetlab.xsd b/sfa/rspecs/resources/ext/planetlab.xsd deleted file mode 100644 index e862877a..00000000 --- a/sfa/rspecs/resources/ext/planetlab.xsd +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - diff --git a/sfa/rspecs/rspec.py b/sfa/rspecs/rspec.py index 00095e2b..50ded822 100755 --- a/sfa/rspecs/rspec.py +++ b/sfa/rspecs/rspec.py @@ -19,9 +19,10 @@ class RSpec: self.elements = {} if rspec: self.parse_xml(rspec) - else: + elif version: self.create(version) - + else: + raise InvalidRSpec("No RSpec or version sepcified. Must specify a valid rspec string or a valid version") def create(self, version=None): """ Create root element diff --git a/sfa/rspecs/version_manager.py b/sfa/rspecs/version_manager.py index 2eabaa49..3e7500ce 100644 --- a/sfa/rspecs/version_manager.py +++ b/sfa/rspecs/version_manager.py @@ -35,7 +35,7 @@ class VersionManager: ### sounds like we should be glad with the first match, not the last one break if not retval: - raise UnvalidRSpecVersion("%s %s is not suported here"% (type, version_num, content_type)) + raise UnsupportedRSpecVersion("[%s %s %s] is not suported here"% (type, version_num, content_type)) return retval def get_version(self, version=None): diff --git a/sfa/rspecs/versions/sfav1.py b/sfa/rspecs/versions/sfav1.py index affab84f..ec279715 100644 --- a/sfa/rspecs/versions/sfav1.py +++ b/sfa/rspecs/versions/sfav1.py @@ -104,17 +104,20 @@ class SFAv1(BaseVersion): return attributes - def add_sliver_attribute(self, hostname, name, value, network=None): - nodes = self.get_nodes({'component_id': '*%s*' % hostname}) - if not nodes: + def add_sliver_attribute(self, component_id, name, value, network=None): + nodes = self.get_nodes({'component_id': '*%s*' % component_id}) + if nodes is not None and isinstance(nodes, list) and len(nodes) > 0: node = nodes[0] slivers = SFAv1Sliver.get_slivers(node) if slivers: sliver = slivers[0] - SFAv1Sliver.add_attribute(sliver, name, value) + SFAv1Sliver.add_sliver_attribute(sliver, name, value) + else: + # should this be an assert / raise an exception? + logger.error("WARNING: failed to find component_id %s" % component_id) - def get_sliver_attributes(self, hostname, network=None): - nodes = self.get_nodes({'component_id': '*%s*' %hostname}) + def get_sliver_attributes(self, component_id, network=None): + nodes = self.get_nodes({'component_id': '*%s*' % component_id}) attribs = [] if nodes is not None and isinstance(nodes, list) and len(nodes) > 0: node = nodes[0] @@ -124,19 +127,24 @@ class SFAv1(BaseVersion): attribs = SFAv1Sliver.get_sliver_attributes(sliver.element) return attribs - def remove_sliver_attribute(self, hostname, name, value, network=None): - attribs = self.get_sliver_attributes(hostname) + def remove_sliver_attribute(self, component_id, name, value, network=None): + attribs = self.get_sliver_attributes(component_id) for attrib in attribs: if attrib['name'] == name and attrib['value'] == value: - attrib.element.delete() + #attrib.element.delete() + parent = attrib.element.getparent() + parent.remove(attrib.element) def add_default_sliver_attribute(self, name, value, network=None): if network: defaults = self.xml.xpath("//network[@name='%s']/sliver_defaults" % network) else: - defaults = self.xml.xpath("//sliver_defaults" % network) - if not defaults : - network_tag = self.xml.xpath("//network[@name='%s']" % network) + defaults = self.xml.xpath("//sliver_defaults") + if not defaults: + if network: + network_tag = self.xml.xpath("//network[@name='%s']" % network) + else: + network_tag = self.xml.xpath("//network") if isinstance(network_tag, list): network_tag = network_tag[0] defaults = network_tag.add_element('sliver_defaults') @@ -153,14 +161,12 @@ class SFAv1(BaseVersion): return SFAv1Sliver.get_sliver_attributes(defaults[0]) def remove_default_sliver_attribute(self, name, value, network=None): - if network: - defaults = self.xml.xpath("//network[@name='%s']/sliver_defaults" % network) - else: - defaults = self.xml.xpath("//sliver_defaults" % network) - attribs = SFAv1Sliver.get_sliver_attributes(defaults) + attribs = self.get_default_sliver_attributes(network) for attrib in attribs: if attrib['name'] == name and attrib['value'] == value: - attrib.element.delete() + #attrib.element.delete() + parent = attrib.element.getparent() + parent.remove(attrib.element) # Links diff --git a/sfa/server/interface.py b/sfa/server/interface.py index 3866a452..2f461461 100644 --- a/sfa/server/interface.py +++ b/sfa/server/interface.py @@ -1,5 +1,4 @@ -#from sfa.util.faults import * -import sfa.client.xmlrpcprotocol as xmlrpcprotocol +from sfa.client.sfaserverproxy import SfaServerProxy from sfa.util.xml import XML # GeniLight client support is optional @@ -31,7 +30,7 @@ class Interface: # xxx url and self.api are undefined server = GeniClientLight(url, self.api.key_file, self.api.cert_file) else: - server = xmlrpcprotocol.server_proxy(self.get_url(), key_file, cert_file, timeout) + server = SfaServerProxy(self.get_url(), key_file, cert_file, timeout) return server ## diff --git a/sfa/server/modpython/SfaAggregateModPython.py b/sfa/server/modpython/SfaAggregateModPython.py index 515b083e..8c97d194 100755 --- a/sfa/server/modpython/SfaAggregateModPython.py +++ b/sfa/server/modpython/SfaAggregateModPython.py @@ -9,7 +9,6 @@ import sys import traceback -import xmlrpclib from mod_python import apache from sfa.util.sfalogging import logger diff --git a/sfa/server/modpython/SfaRegistryModPython.py b/sfa/server/modpython/SfaRegistryModPython.py index 1be54807..6ea2710b 100755 --- a/sfa/server/modpython/SfaRegistryModPython.py +++ b/sfa/server/modpython/SfaRegistryModPython.py @@ -9,7 +9,6 @@ import sys import traceback -import xmlrpclib from mod_python import apache from sfa.util.sfalogging import logger diff --git a/sfa/server/modpython/SfaSliceMgrModPython.py b/sfa/server/modpython/SfaSliceMgrModPython.py index cd655a7e..7d0e5f2c 100755 --- a/sfa/server/modpython/SfaSliceMgrModPython.py +++ b/sfa/server/modpython/SfaSliceMgrModPython.py @@ -9,7 +9,6 @@ import sys import traceback -import xmlrpclib from mod_python import apache from sfa.util.sfalogging import logger diff --git a/sfa/server/sfa-clean-peer-records.py b/sfa/server/sfa-clean-peer-records.py index 62acb4b2..bcb917bc 100644 --- a/sfa/server/sfa-clean-peer-records.py +++ b/sfa/server/sfa-clean-peer-records.py @@ -14,7 +14,7 @@ from sfa.server.registry import Registries from sfa.storage.table import SfaTable -import sfa.client.xmlrpcprotocol as xmlrpcprotocol +from sfa.client.sfaserverproxy import SfaServerProxy from sfa.generic import Generic @@ -35,7 +35,7 @@ def main(): # and a valid credential authority = config.SFA_INTERFACE_HRN url = 'http://%s:%s/' %(config.SFA_REGISTRY_HOST, config.SFA_REGISTRY_PORT) - registry = xmlrpcprotocol.server_proxy(url, key_file, cert_file) + registry = SfaServerProxy(url, key_file, cert_file) sfa_api = Generic.the_flavour() credential = sfa_api.getCredential() diff --git a/sfa/server/sfa-start.py b/sfa/server/sfa-start.py index 6c050601..8f6faf24 100755 --- a/sfa/server/sfa-start.py +++ b/sfa/server/sfa-start.py @@ -25,10 +25,6 @@ # TODO: Can all three servers use the same "registry" certificate? ## -# TCP ports for the three servers -#registry_port=12345 -#aggregate_port=12346 -#slicemgr_port=12347 ### xxx todo not in the config yet component_port=12346 import os, os.path @@ -39,7 +35,6 @@ from optparse import OptionParser from sfa.util.sfalogging import logger from sfa.util.xrn import get_authority, hrn_to_urn from sfa.util.config import Config -import sfa.client.xmlrpcprotocol as xmlrpcprotocol from sfa.trust.gid import GID from sfa.trust.trustedroots import TrustedRoots from sfa.trust.certificate import Keypair, Certificate @@ -47,7 +42,6 @@ from sfa.trust.hierarchy import Hierarchy from sfa.trust.gid import GID from sfa.server.sfaapi import SfaApi - from sfa.server.registry import Registries from sfa.server.aggregate import Aggregates diff --git a/sfa/server/sfa_component_setup.py b/sfa/server/sfa_component_setup.py index 3775391a..589a5720 100755 --- a/sfa/server/sfa_component_setup.py +++ b/sfa/server/sfa_component_setup.py @@ -6,7 +6,7 @@ from optparse import OptionParser from sfa.util.faults import ConnectionKeyGIDMismatch from sfa.util.config import Config -import sfa.client.xmlrpcprotocol as xmlrpcprotocol +from sfa.client.sfaserverproxy import SfaServerProxy from sfa.util.plxrn import hrn_to_pl_slicename, slicename_to_hrn from sfa.trust.certificate import Keypair, Certificate @@ -48,7 +48,7 @@ def server_proxy(url=None, port=None, keyfile=None, certfile=None,verbose=False) if verbose: print "Contacting registry at: %(url)s" % locals() - server = xmlrpcprotocol.server_proxy(url, keyfile, certfile) + server = SfaServerProxy(url, keyfile, certfile) return server diff --git a/sfa/server/sfaapi.py b/sfa/server/sfaapi.py index 79a5540d..c18452b9 100644 --- a/sfa/server/sfaapi.py +++ b/sfa/server/sfaapi.py @@ -249,9 +249,8 @@ class SfaApi (XmlrpcApi): Converts the specified result into a standard GENI compliant response """ + # as of dec 13 2011 we only support API v2 if self.interface.lower() in ['aggregate', 'slicemgr']: - if hasattr(self.config, 'SFA_AGGREGATE_API_VERSION') and \ - self.config.SFA_AGGREGATE_API_VERSION == 2: - result = self.prepare_response_v2_am(result) + result = self.prepare_response_v2_am(result) return XmlrpcApi.prepare_response(self, result, method) diff --git a/sfa/server/threadedserver.py b/sfa/server/threadedserver.py index 7a9c368b..1a78f35a 100644 --- a/sfa/server/threadedserver.py +++ b/sfa/server/threadedserver.py @@ -11,6 +11,7 @@ import socket import traceback import threading from Queue import Queue +import xmlrpclib import SocketServer import BaseHTTPServer import SimpleXMLRPCServer @@ -42,6 +43,7 @@ def verify_callback(conn, x509, err, depth, preverify): # and ignore them # XXX SMBAKER: I don't know what this error is, but it's being returned + # xxx thierry: this most likely means the cert has a validity range in the future # by newer pl nodes. if err == 9: #print " X509_V_ERR_CERT_NOT_YET_VALID" @@ -71,7 +73,7 @@ def verify_callback(conn, x509, err, depth, preverify): #print " X509_V_ERR_CERT_UNTRUSTED" return 1 - print " error", err, "in verify_callback" + logger.debug(" error %s in verify_callback"%err) return 0 diff --git a/sfa/storage/sfa.sql b/sfa/storage/sfa.sql index 3d7984d1..9a2792c8 100644 --- a/sfa/storage/sfa.sql +++ b/sfa/storage/sfa.sql @@ -36,20 +36,33 @@ CREATE AGGREGATE array_accum ( initcond = '{}' ); +-- Valid record types +CREATE TABLE record_types ( + record_type text PRIMARY KEY +) WITH OIDS; +INSERT INTO record_types (record_type) VALUES ('authority'); +INSERT INTO record_types (record_type) VALUES ('authority+sa'); +INSERT INTO record_types (record_type) VALUES ('authority+am'); +INSERT INTO record_types (record_type) VALUES ('authority+sm'); +INSERT INTO record_types (record_type) VALUES ('user'); +INSERT INTO record_types (record_type) VALUES ('slice'); +INSERT INTO record_types (record_type) VALUES ('node'); + + -- main table -CREATE TABLE sfa ( +CREATE TABLE records ( record_id serial PRIMARY KEY , hrn text NOT NULL, authority text NOT NULL, peer_authority text, gid text, - type text NOT NULL, + type text REFERENCES record_types, pointer integer, date_created timestamp without time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, last_updated timestamp without time zone NOT NULL DEFAULT CURRENT_TIMESTAMP ); -CREATE INDEX sfa_hrn_ids on sfa (hrn); -CREATE INDEX sfa_type_ids on sfa (type); -CREATE INDEX sfa_authority_ids on sfa (authority); -CREATE INDEX sfa_peer_authority_ids on sfa (peer_authority); -CREATE INDEX sfa_pointer_ids on sfa (pointer); +CREATE INDEX sfa_hrn_ids on records (hrn); +CREATE INDEX sfa_type_ids on records (type); +CREATE INDEX sfa_authority_ids on records (authority); +CREATE INDEX sfa_peer_authority_ids on records (peer_authority); +CREATE INDEX sfa_pointer_ids on records (pointer); diff --git a/sfa/storage/table.py b/sfa/storage/table.py index acd0b3d9..602bcabe 100644 --- a/sfa/storage/table.py +++ b/sfa/storage/table.py @@ -14,7 +14,7 @@ from sfa.storage.record import SfaRecord, AuthorityRecord, NodeRecord, SliceReco class SfaTable(list): - SFA_TABLE_PREFIX = "sfa" + SFA_TABLE_PREFIX = "records" def __init__(self, record_filter = None): diff --git a/sfa/util/cache.py b/sfa/util/cache.py index a2ded4a4..ee4716c6 100644 --- a/sfa/util/cache.py +++ b/sfa/util/cache.py @@ -8,7 +8,7 @@ import pickle from datetime import datetime # maximum lifetime of cached data (in seconds) -MAX_CACHE_TTL = 60 * 60 +DEFAULT_CACHE_TTL = 60 * 60 class CacheData: @@ -17,7 +17,7 @@ class CacheData: expires = None lock = None - def __init__(self, data, ttl = MAX_CACHE_TTL): + def __init__(self, data, ttl = DEFAULT_CACHE_TTL): self.lock = threading.RLock() self.data = data self.renew(ttl) @@ -31,11 +31,11 @@ class CacheData: def get_expires_date(self): return str(datetime.fromtimestamp(self.expires)) - def renew(self, ttl = MAX_CACHE_TTL): + def renew(self, ttl = DEFAULT_CACHE_TTL): self.created = time.time() self.expires = self.created + ttl - def set_data(self, data, renew=True, ttl = MAX_CACHE_TTL): + def set_data(self, data, renew=True, ttl = DEFAULT_CACHE_TTL): with self.lock: self.data = data if renew: @@ -73,7 +73,7 @@ class Cache: if filename: self.load_from_file(filename) - def add(self, key, value, ttl = MAX_CACHE_TTL): + def add(self, key, value, ttl = DEFAULT_CACHE_TTL): with self.lock: if self.cache.has_key(key): self.cache[key].set_data(value, ttl=ttl) diff --git a/sfa/util/method.py b/sfa/util/method.py index 4bfeb9bb..fc2ebaac 100644 --- a/sfa/util/method.py +++ b/sfa/util/method.py @@ -245,8 +245,7 @@ class Method: elif not isinstance(value, expected_type): raise SfaInvalidArgument("expected %s, got %s" % \ - (xmlrpc_type(expected_type), - xmlrpc_type(type(value))), + (xmlrpc_type(expected_type), xmlrpc_type(type(value))), name) # If a minimum or maximum (length, value) has been specified diff --git a/sfa/util/plxrn.py b/sfa/util/plxrn.py index f292823d..17d0c15b 100644 --- a/sfa/util/plxrn.py +++ b/sfa/util/plxrn.py @@ -59,8 +59,10 @@ class PlXrn (Xrn): def pl_slicename (self): self._normalize() leaf = self.leaf - leaf = re.sub('[^a-zA-Z0-9_]', '', leaf) - return self.pl_login_base() + '_' + leaf + sliver_id_parts = leaf.split(':') + name = sliver_id_parts[0] + name = re.sub('[^a-zA-Z0-9_]', '', name) + return self.pl_login_base() + '_' + name #def hrn_to_pl_authname(hrn): def pl_authname (self): diff --git a/sfa/util/storage.py b/sfa/util/storage.py index ee2e41b8..793a38eb 100644 --- a/sfa/util/storage.py +++ b/sfa/util/storage.py @@ -1,5 +1,6 @@ import os from sfa.util.xml import XML + class SimpleStorage(dict): """ Handles storing and loading python dictionaries. The storage file created diff --git a/sfa/util/xml.py b/sfa/util/xml.py index 1a7f2f37..90693f2e 100755 --- a/sfa/util/xml.py +++ b/sfa/util/xml.py @@ -195,7 +195,7 @@ class XML: self.namespaces['default'] = 'default' self.root = XmlElement(root, self.namespaces) - # set schema + # set schema for key in self.root.attrib.keys(): if key.endswith('schemaLocation'): # schema location should be at the end of the list @@ -208,7 +208,7 @@ class XML: if element is None: if self.root is None: self.parse_xml('<%s/>' % root_tag_name) - element = self.root + element = self.root.element if 'text' in d: text = d.pop('text') @@ -224,9 +224,9 @@ class XML: self.parse_dict(val, key, child_element) elif isinstance(val, basestring): child_element = etree.SubElement(element, key).text = val - + elif isinstance(value, int): - d[key] = unicode(d[key]) + d[key] = unicode(d[key]) elif value is None: d.pop(key) diff --git a/tests/testGid.py b/tests/testGid.py index 85f92407..36b91fea 100755 --- a/tests/testGid.py +++ b/tests/testGid.py @@ -1,5 +1,4 @@ import unittest -import xmlrpclib from sfa.trust.certificate import Keypair from sfa.trust.gid import * diff --git a/tests/testInterfaces.py b/tests/testInterfaces.py index fb57a717..a51799b8 100755 --- a/tests/testInterfaces.py +++ b/tests/testInterfaces.py @@ -4,7 +4,7 @@ import os import random import string import unittest -import sfa.util.xmlrpcprotocol as xmlrpcprotocol + from unittest import TestCase from optparse import OptionParser from sfa.util.xrn import get_authority @@ -13,6 +13,7 @@ from sfa.trust.certificate import * from sfa.trust.credential import * from sfa.trust.sfaticket import SfaTicket from sfa.client import sfi +from sfa.client.sfaserverproxy import SfaServerProxy, ServerException def random_string(size): return "".join(random.sample(string.letters, size)) @@ -43,10 +44,10 @@ class Client: self.cert.save_to_file(cert_file) SFI_AGGREGATE = config.SFI_SM.replace('12347', '12346') SFI_CM = 'http://' + options.cm_host + ':12346' - self.registry = xmlrpcprotocol.server_proxy(config.SFI_REGISTRY, key_file, cert_file) - self.aggregate = xmlrpcprotocol.server_proxy(SFI_AGGREGATE, key_file, cert_file) - self.sm = xmlrpcprotocol.server_proxy(config.SFI_SM, key_file, cert_file) - self.cm = xmlrpcprotocol.server_proxy(SFI_CM, key_file, cert_file) + self.registry = SfaServerProxy(config.SFI_REGISTRY, key_file, cert_file) + self.aggregate = SfaServerProxy(SFI_AGGREGATE, key_file, cert_file) + self.sm = SfaServerProxy(config.SFI_SM, key_file, cert_file) + self.cm = SfaServerProxy(SFI_CM, key_file, cert_file) self.hrn = config.SFI_USER # XX defaulting to user, but this should be configurable so we can # test from components persepctive @@ -170,7 +171,7 @@ class RegistryTest(BasicTestCase): server_exception = False try: callable(self.credential) - except xmlrpcprotocol.ServerException: + except ServerException: server_exception = True finally: if self.type in ['user'] and not server_exception: diff --git a/tests/testKeypair.py b/tests/testKeypair.py index ebe1edec..424740ae 100755 --- a/tests/testKeypair.py +++ b/tests/testKeypair.py @@ -3,7 +3,6 @@ import sys sys.path.append('..') import unittest -import xmlrpclib import base64 from sfa.trust.certificate import Keypair diff --git a/tests/testRecord.py b/tests/testRecord.py index 329d84e9..f931bdad 100755 --- a/tests/testRecord.py +++ b/tests/testRecord.py @@ -1,5 +1,4 @@ import unittest -import xmlrpclib from sfa.trust.gid import * from sfa.util.config import * from sfa.storage.record import *