From: Tony Mack Date: Thu, 7 Nov 2013 22:37:51 +0000 (-0500) Subject: Merge branch 'geni-v3' of ssh://git.planet-lab.org/git/sfa into geni-v3 X-Git-Tag: sfa-3.1-1~45^2~1 X-Git-Url: http://git.onelab.eu/?p=sfa.git;a=commitdiff_plain;h=f58accad0a467774fa451ef690147d0b93671f44;hp=7131d06e07dff99df436f0fdc2ad7bae77bd62bf Merge branch 'geni-v3' of ssh://git.planet-lab.org/git/sfa into geni-v3 --- diff --git a/clientbin/sfadump.py b/clientbin/sfadump.py index 57fdf790..93fca705 100755 --- a/clientbin/sfadump.py +++ b/clientbin/sfadump.py @@ -5,9 +5,10 @@ import sys import os, os.path import tempfile from types import StringTypes, ListType -from optparse import OptionParser +from argparse import ArgumentParser from sfa.util.sfalogging import logger +from sfa.util.faults import CredentialNotVerifiable, CertMissingParent #, ChildRightsNotSubsetOfParent from sfa.trust.certificate import Certificate from sfa.trust.credential import Credential @@ -73,50 +74,69 @@ def extract_gids(cred, extract_parents): # if parent: # extract_gids(parent, extract_parents) +def verify_input_object (obj, kind, options): + if options.trusted_roots: + print "CHECKING...", + message= "against [" + (" + ".join(options.trusted_roots)) + "]" + try: + if kind=='credential': + print "verify",message, + obj.verify(options.trusted_roots) + elif kind in ['certificate','gid']: + print "verify_chain",message, + obj.verify_chain(options.trusted_roots) + print "--> OK" + except Exception as inst: + print "--> KO",type(inst).__name__ + def handle_input (filename, options): kind = determine_sfa_filekind(filename) - handle_input_kind (filename,options,kind) - -def handle_input_kind (filename, options, kind): - -# dump methods current do 'print' so let's go this road for now + # dump methods current do 'print' so let's go this road for now if kind=="certificate": cert=Certificate (filename=filename) print '--------------------',filename,'IS A',kind cert.dump(show_extensions=options.show_extensions) + verify_input_object (cert, kind, options) elif kind=="credential": cred = Credential(filename = filename) print '--------------------',filename,'IS A',kind cred.dump(dump_parents = options.dump_parents, show_xml=options.show_xml) if options.extract_gids: - print '--------------------',filename,'embedded GIDS' + print '--------------------',filename,'embedded GIDs' extract_gids(cred, extract_parents = options.dump_parents) + verify_input_object (cred, kind, options) elif kind=="gid": gid = GID(filename = filename) print '--------------------',filename,'IS A',kind gid.dump(dump_parents = options.dump_parents) + verify_input_object (gid, kind, options) else: print "%s: unknown filekind '%s'"% (filename,kind) def main(): usage = """%prog file1 [ .. filen] display info on input files""" - parser = OptionParser(usage=usage) - - parser.add_option("-g", "--extract-gids", action="store_true", dest="extract_gids", default=False, help="Extract GIDs from credentials") - parser.add_option("-p", "--dump-parents", action="store_true", dest="dump_parents", default=False, help="Show parents") - parser.add_option("-e", "--extensions", action="store_true", dest="show_extensions", default="False", help="Show certificate extensions") - parser.add_option("-v", "--verbose", action='count', dest='verbose', default=0, help="More and more verbose") - parser.add_option("-x", "--xml", action='store_true', dest='show_xml', default=False, help="dumps xml tree (cred. only)") - (options, args) = parser.parse_args() + parser = ArgumentParser(usage=usage) + + parser.add_argument("-g", "--extract-gids", action="store_true", dest="extract_gids", + default=False, help="Extract GIDs from credentials") + parser.add_argument("-p", "--dump-parents", action="store_true", dest="dump_parents", + default=False, help="Show parents") + parser.add_argument("-e", "--extensions", action="store_true", + dest="show_extensions", default="False", help="Show certificate extensions") + parser.add_argument("-v", "--verbose", action='count', + dest='verbose', default=0, help="More and more verbose") + parser.add_argument("-x", "--xml", action='store_true', + dest='show_xml', default=False, help="dumps xml tree (cred. only)") + parser.add_argument("-c", "--check", action='append', dest='trusted_roots', + help="cumulative list of trusted GIDs - when provided, the input is verify'ed against these") + parser.add_argument("filenames",metavar='F',nargs='+',help="filenames to dump") + options = parser.parse_args() logger.setLevelFromOptVerbose(options.verbose) - if len(args) <= 0: - parser.print_help() - sys.exit(1) - for f in args: - handle_input(f,options) + for filename in options.filenames: + handle_input(filename,options) if __name__=="__main__": main() diff --git a/debian/sfa-senslab.install b/debian/sfa-senslab.install index c445dc0f..25c02099 100644 --- a/debian/sfa-senslab.install +++ b/debian/sfa-senslab.install @@ -1 +1 @@ -usr/lib*/python*/site-packages/sfa/senslab +usr/lib*/python*/site-packages/sfa/iotlab diff --git a/init.d/sfa b/init.d/sfa index db1708f7..0ba2385a 100755 --- a/init.d/sfa +++ b/init.d/sfa @@ -315,7 +315,7 @@ function start() { [ "$SFA_SM_ENABLED" == 1 -o "$SFA_SM_ENABLED" == True ] && action "SFA: SliceMgr" daemon /usr/bin/sfa-start.py -s -d $OPTIONS - [ "$SFA_FLASHPOLICY_ENABLED" == 1 ] && \ + [ "$SFA_FLASHPOLICY_ENABLED" == 1 -o "$SFA_FLASHPOLICY_ENABLED" == True ] && \ action "Flash Policy Server" daemon /usr/bin/sfa_flashpolicy.py --file="$SFA_FLASHPOLICY_CONFIG_FILE" --port=$SFA_FLASHPOLICY_PORT -d touch $SFALOCK @@ -361,4 +361,3 @@ case "$1" in esac exit $RETVAL - diff --git a/setup.py b/setup.py index e6c041aa..162d0577 100755 --- a/setup.py +++ b/setup.py @@ -16,22 +16,22 @@ except: version_tag='cleaningup' scripts = glob("clientbin/*.py") + \ - [ + [ 'config/sfa-config-tty', 'config/sfa-config', # 'config/gen-sfa-cm-config.py', - 'sfa/server/sfa-start.py', -# 'sfa/server/sfa_component_setup.py', + 'sfa/server/sfa-start.py', +# 'sfa/server/sfa_component_setup.py', 'sfatables/sfatables', 'keyconvert/keyconvert.py', 'flashpolicy/sfa_flashpolicy.py', ] packages = [ - 'sfa', + 'sfa', 'sfa/trust', 'sfa/storage', - 'sfa/util', + 'sfa/util', 'sfa/server', 'sfa/methods', 'sfa/generic', @@ -48,7 +48,7 @@ packages = [ 'sfa/dummy', 'sfa/openstack', 'sfa/federica', - 'sfa/senslab', + 'sfa/iotlab', 'sfatables', 'sfatables/commands', 'sfatables/processors', @@ -96,24 +96,24 @@ if sys.argv[1] in ['uninstall', 'remove', 'delete', 'clean']: remove_bins = [ '/usr/bin/' + os.path.basename(bin) for bin in scripts ] remove_files = remove_bins + [ "/etc/init.d/%s"%x for x in initscripts ] - # remove files + # remove files for filepath in remove_files: print "removing", filepath, "...", - try: + try: os.remove(filepath) print "success" except: print "failed" - # remove directories - for directory in remove_dirs: + # remove directories + for directory in remove_dirs: print "removing", directory, "...", - try: + try: shutil.rmtree(directory) print "success" except: print "failed" else: # avoid repeating what's in the specfile already setup(name='sfa', - packages = packages, + packages = packages, data_files = data_files, scripts = scripts, url="http://svn.planet-lab.org/wiki/SFATutorial", diff --git a/sfa.spec b/sfa.spec index 665d7c7e..e8f8dafe 100644 --- a/sfa.spec +++ b/sfa.spec @@ -1,6 +1,6 @@ %define name sfa %define version 3.0 -%define taglevel 0 +%define taglevel 2 %define release %{taglevel}%{?pldistro:.%{pldistro}}%{?date:.%{date}} %global python_sitearch %( python -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)" ) @@ -222,7 +222,7 @@ rm -rf $RPM_BUILD_ROOT %{python_sitelib}/sfa/nitos %files senslab -%{python_sitelib}/sfa/senslab +%{python_sitelib}/sfa/iotlab %files dummy %{python_sitelib}/sfa/dummy @@ -265,6 +265,39 @@ fi #[ "$1" -ge "1" ] && service sfa-cm restart || : %changelog +* Thu Oct 10 2013 Thierry Parmentelat - sfa-3.0-2 +- -- core +- Extend all versions of rspecs in order to support "links" and "channels" management methods +- several fixes aroung Rspecs in order to make the code usable by 3rd-party tools +- expose PL node tags in PGv2 and GENIv3 rspecs +- fix default slivers mgt in sfav1 rspecs +- fix SM and AM urn format in GetVersion +- fix sfaadmin.py to handle extra testbed-dependent info +- -- PlanetLab +- Better management of external Slices/Persons/Sites +- Importer ignores external Slices/Persons/Sites +- -- Nitos +- uniformize Leases management +- address channels with urn +- -- IotLab +- Renaming slab folders, files and variables to iotlab, the new name of the platform. +- New class ParsingResourcesFull in Oarrestapi file. Contain all the parsing nodes functions. +- Adding a mobile field to iotlab rspec and a mobility type attribute . +- Granularity changed to 60 sec. +- Fixing returned rspec in GetSlices and GetLeases. +- Corrections in import (importing nodes) +- More testis in script files under testbeds/iotlab/tests . Creation of Rspecs request models under r testbeds/iotlab/tests /tests_rspec +- Lease filtering by date added in GetLeases. +- Commenting and cleaning + +* Wed Jun 19 2013 Thierry Parmentelat - sfa-3.0-1 +- AM API v3-based first implementation. +- planetlab native V3 driver +- dummy native V3 driver +- introduction of v2_to_v3_adapter +- lease times in RFC3339 date format, not epoch +- mostly in line with geni-v2 for non-v3 issues like packaging and all + * Tue Feb 26 2013 Thierry Parmentelat - sfa-2.1-25 - sfi and sfaadmin list now share the same display code for related objs - support for advertising alternate api urls - for other API versions - api_versions.xml diff --git a/sfa/client/client_helper.py b/sfa/client/client_helper.py index 48160a49..15f4a2db 100644 --- a/sfa/client/client_helper.py +++ b/sfa/client/client_helper.py @@ -19,8 +19,8 @@ def pg_users_arg(records): for record in records: if record['type'] != 'user': continue - user = {'urn': record['geni_urn'], - 'keys': record['keys'], + user = {'urn': record['reg-urn'], + 'keys': record['reg-keys'], 'email': record['email']} users.append(user) return users diff --git a/sfa/client/sfaadmin.py b/sfa/client/sfaadmin.py index de247bdd..b7b45d46 100755 --- a/sfa/client/sfaadmin.py +++ b/sfa/client/sfaadmin.py @@ -15,7 +15,7 @@ from sfa.trust.certificate import convert_public_key from sfa.client.candidates import Candidates -from sfa.client.common import optparse_listvalue_callback, terminal_render, filter_records +from sfa.client.common import optparse_listvalue_callback, optparse_dictvalue_callback, terminal_render, filter_records pprinter = PrettyPrinter(indent=4) @@ -84,7 +84,7 @@ class RegistryCommands(Commands): def _record_dict(self, xrn=None, type=None, url=None, description=None, email='', key=None, - slices=[], researchers=[], pis=[]): + slices=[], researchers=[], pis=[], extras={}): record_dict = {} if xrn: if type: @@ -112,6 +112,8 @@ class RegistryCommands(Commands): record_dict['email'] = email if pis: record_dict['pi'] = pis + if extras: + record_dict.update(extras) return record_dict @@ -200,11 +202,12 @@ Users having a GID/PubKey correpondence NOT OK: %s and are: \n%s\n\n"%(len(NOKEY @args('-p', '--pis', dest='pis', metavar='', help='Set/replace Principal Investigators/Project Managers', default='', type="str", action='callback', callback=optparse_listvalue_callback) + @args('-X','--extra',dest='extras',default={},type='str',metavar="", action="callback", callback=optparse_dictvalue_callback, nargs=1, help="set extra/testbed-dependent flags, e.g. --extra enabled=true") def register(self, xrn, type=None, url=None, description=None, key=None, slices='', - pis='', researchers='',email=''): + pis='', researchers='',email='', extras={}): """Create a new Registry record""" record_dict = self._record_dict(xrn=xrn, type=type, url=url, key=key, - slices=slices, researchers=researchers, email=email, pis=pis) + slices=slices, researchers=researchers, email=email, pis=pis, extras=extras) self.api.manager.Register(self.api, record_dict) @@ -222,12 +225,13 @@ Users having a GID/PubKey correpondence NOT OK: %s and are: \n%s\n\n"%(len(NOKEY @args('-p', '--pis', dest='pis', metavar='', help='Set/replace Principal Investigators/Project Managers', default='', type="str", action='callback', callback=optparse_listvalue_callback) + @args('-X','--extra',dest='extras',default={},type='str',metavar="", action="callback", callback=optparse_dictvalue_callback, nargs=1, help="set extra/testbed-dependent flags, e.g. --extra enabled=true") def update(self, xrn, type=None, url=None, description=None, key=None, slices='', - pis='', researchers=''): + pis='', researchers='', extras={}): """Update an existing Registry record""" print 'incoming PIS',pis record_dict = self._record_dict(xrn=xrn, type=type, url=url, description=description, - key=key, slices=slices, researchers=researchers, pis=pis) + key=key, slices=slices, researchers=researchers, pis=pis, extras=extras) self.api.manager.Update(self.api, record_dict) @args('-x', '--xrn', dest='xrn', metavar='', help='object hrn/urn (mandatory)') @@ -256,7 +260,7 @@ Users having a GID/PubKey correpondence NOT OK: %s and are: \n%s\n\n"%(len(NOKEY """Initialize or upgrade the db""" from sfa.storage.dbschema import DBSchema dbschema=DBSchema() - dbschema.init_or_upgrade + dbschema.init_or_upgrade() @args('-a', '--all', dest='all', metavar='', action='store_true', default=False, help='Remove all registry records and all files in %s area' % help_basedir) diff --git a/sfa/client/sfaclientlib.py b/sfa/client/sfaclientlib.py index 2f4cda4f..f4fe3a8d 100644 --- a/sfa/client/sfaclientlib.py +++ b/sfa/client/sfaclientlib.py @@ -9,6 +9,7 @@ import sys import os,os.path +import subprocess from datetime import datetime from sfa.util.xrn import Xrn @@ -24,7 +25,7 @@ from sfa.trust.certificate import Keypair, Certificate from sfa.trust.credential import Credential from sfa.trust.gid import GID ########## -# a helper class to implement the bootstrapping of crypto. material +# a helper class to implement the bootstrapping of cryptoa. material # assuming we are starting from scratch on the client side # what's needed to complete a full slice creation cycle # (**) prerequisites: @@ -52,7 +53,11 @@ from sfa.trust.gid import GID # obtained at the registry with Resolve # using the (step2) user-credential as credential # default filename is ..cred - +# +# (**) additionnally, it might make sense to upgrade a GID file +# into a pkcs12 certificate usable in a browser +# this bundled format allows for embedding the private key +# ########## Implementation notes # @@ -88,6 +93,11 @@ from sfa.trust.gid import GID # for Java is documented below # http://nam.ece.upatras.gr/fstoolkit/trac/wiki/JavaSFAClient # +# (*) pkcs12 +# +# the implementation of the pkcs12 wrapping, which is a late addition, +# is done through direct calls to openssl +# #################### class SfaClientException (Exception): pass @@ -194,6 +204,22 @@ class SfaClientBootstrap: return output +# http://trac.myslice.info/wiki/MySlice/Developer/SFALogin +### produce a pkcs12 bundled certificate from GID and private key +# xxx for now we put a hard-wired password that's just, well, 'password' +# when leaving this empty on the mac, result can't seem to be loaded in keychain.. + def my_pkcs12_produce (self, filename): + password=raw_input("Enter password for p12 certificate: ") + openssl_command=['openssl', 'pkcs12', "-export"] + openssl_command += [ "-password", "pass:%s"%password ] + openssl_command += [ "-inkey", self.private_key_filename()] + openssl_command += [ "-in", self.my_gid_filename()] + openssl_command += [ "-out", filename ] + if subprocess.call(openssl_command) ==0: + print "Successfully created %s"%filename + else: + print "Failed to create %s"%filename + # Returns True if credential file is valid. Otherwise return false. def validate_credential(self, filename): valid = True @@ -260,7 +286,8 @@ class SfaClientBootstrap: return self.gid_filename (self.hrn, "user") def gid_filename (self, hrn, type): return self.fullpath ("%s.%s.gid"%(hrn,type)) - + def my_pkcs12_filename (self): + return self.fullpath ("%s.p12"%self.hrn) # optimizing dependencies # originally we used classes GID or Credential or Certificate @@ -327,6 +354,9 @@ class SfaClientBootstrap: @get_or_produce (my_gid_filename, my_gid_produce) def my_gid (self): pass + @get_or_produce (my_pkcs12_filename, my_pkcs12_produce) + def my_pkcs12 (self): pass + @get_or_produce (credential_filename, credential_produce, validate_credential) def credential (self, hrn, type): pass diff --git a/sfa/client/sfi.py b/sfa/client/sfi.py index 410e4f8b..dcd16880 100644 --- a/sfa/client/sfi.py +++ b/sfa/client/sfi.py @@ -420,7 +420,7 @@ class Sfi: "authority in set of credentials for this call") # show_credential option - if command in ("list","resources", "describe", "provision", "allocate", "add","update","remove","slices","delete","status","renew"): + if command in ("list","resources", "describe", "provision", "allocate", "add","update","remove","delete","status","renew"): parser.add_option("-C","--credential",dest='show_credential',action='store_true',default=False, help="show credential(s) used in human-readable form") # registy filter option @@ -492,6 +492,8 @@ use this if you mean an authority instead""") metavar="slice_hrn", help="delegate cred. for slice HRN") parser.add_option("-a", "--auths", dest='delegate_auths',action='append',default=[], metavar='auth_hrn', help="delegate PI cred for auth HRN") + parser.add_option('-d', '--delegate', dest='delegate', help="Override 'delegate' from the config file") + parser.add_option('-b', '--backend', dest='backend', help="Override 'backend' from the config file") return parser @@ -667,7 +669,7 @@ use this if you mean an authority instead""") self.private_key = client_bootstrap.private_key() self.my_credential_string = client_bootstrap.my_credential_string () self.my_credential = {'geni_type': 'geni_sfa', - 'geni_version': '3.0', + 'geni_version': '3', 'geni_value': self.my_credential_string} self.my_gid = client_bootstrap.my_gid () self.client_bootstrap = client_bootstrap @@ -687,7 +689,7 @@ use this if you mean an authority instead""") def slice_credential(self, name): return {'geni_type': 'geni_sfa', - 'geni_version': '3.0', + 'geni_version': '3', 'geni_value': self.slice_credential_string(name)} # xxx should be supported by sfaclientbootstrap as well @@ -1047,25 +1049,6 @@ use this if you mean an authority instead""") # Slice-related commands # ================================================================== - @register_command("","") - def slices(self, options, args): - "list instantiated slices (ListSlices) - returns urn's" - server = self.sliceapi() - # creds - creds = [self.my_credential_string] - # options and call_id when supported - api_options = {} - api_options['call_id']=unique_call_id() - if options.show_credential: - show_credentials(creds) - result = server.ListSlices(creds, *self.ois(server,api_options)) - value = ReturnValue.get_value(result) - if self.options.raw: - save_raw_to_file(result, self.options.raw, self.options.rawformat, self.options.rawbanner) - else: - display_list(value) - return - # show rspec for named slice @register_command("","") def resources(self, options, args): @@ -1104,9 +1087,9 @@ use this if you mean an authority instead""") # just request the version the client wants api_options['geni_rspec_version'] = version_manager.get_version(options.rspec_version).to_dict() else: - api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3.0'} + api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3'} else: - api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3.0'} + api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3'} result = server.ListResources (creds, api_options) value = ReturnValue.get_value(result) if self.options.raw: @@ -1137,7 +1120,7 @@ use this if you mean an authority instead""") 'cached': True, 'info': options.info, 'list_leases': options.list_leases, - 'geni_rspec_version': {'type': 'geni', 'version': '3.0'}, + 'geni_rspec_version': {'type': 'geni', 'version': '3'}, } if options.rspec_version: version_manager = VersionManager() @@ -1146,7 +1129,7 @@ use this if you mean an authority instead""") # just request the version the client wants api_options['geni_rspec_version'] = version_manager.get_version(options.rspec_version).to_dict() else: - api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3.0'} + api_options['geni_rspec_version'] = {'type': 'geni', 'version': '3'} urn = Xrn(args[0], type='slice').get_urn() result = server.Describe([urn], creds, api_options) value = ReturnValue.get_value(result) @@ -1218,6 +1201,21 @@ use this if you mean an authority instead""") rspec = open(rspec_file).read() api_options = {} api_options ['call_id'] = unique_call_id() + # users + sfa_users = [] + geni_users = [] + slice_records = self.registry().Resolve(slice_urn, [self.my_credential_string]) + if slice_records and 'reg-researchers' in slice_records[0] and slice_records[0]['reg-researchers']!=[]: + slice_record = slice_records[0] + user_hrns = slice_record['reg-researchers'] + user_urns = [hrn_to_urn(hrn, 'user') for hrn in user_hrns] + user_records = self.registry().Resolve(user_urns, [self.my_credential_string]) + sfa_users = sfa_users_arg(user_records, slice_record) + geni_users = pg_users_arg(user_records) + + api_options['sfa_users'] = sfa_users + api_options['geni_users'] = geni_users + result = server.Allocate(slice_urn, creds, rspec, api_options) value = ReturnValue.get_value(result) if self.options.raw: @@ -1259,7 +1257,7 @@ use this if you mean an authority instead""") # set the requtested rspec version version_manager = VersionManager() - rspec_version = version_manager._get_version('geni', '3.0').to_dict() + rspec_version = version_manager._get_version('geni', '3').to_dict() api_options['geni_rspec_version'] = rspec_version # users @@ -1270,9 +1268,9 @@ use this if you mean an authority instead""") # }] users = [] slice_records = self.registry().Resolve(slice_urn, [self.my_credential_string]) - if slice_records and 'researcher' in slice_records[0] and slice_records[0]['researcher']!=[]: + if slice_records and 'reg-researchers' in slice_records[0] and slice_records[0]['reg-researchers']!=[]: slice_record = slice_records[0] - user_hrns = slice_record['researcher'] + user_hrns = slice_record['reg-researchers'] user_urns = [hrn_to_urn(hrn, 'user') for hrn in user_hrns] user_records = self.registry().Resolve(user_urns, [self.my_credential_string]) users = pg_users_arg(user_records) @@ -1495,8 +1493,9 @@ $ sfi myslice $ sfi -v myslice -- or sfi -vv myslice same but with more and more verbosity -$ sfi m +$ sfi m -b http://mymanifold.foo.com:7080/ is synonym to sfi myslice as no other command starts with an 'm' + and uses a custom backend for this one call """ ) # register_command def myslice (self, options, args): @@ -1512,21 +1511,31 @@ $ sfi m if len(args)>0: self.print_help() sys.exit(1) + # enable info by default + self.logger.setLevelFromOptVerbose(self.options.verbose+1) ### the rough sketch goes like this + # (0) produce a p12 file + self.client_bootstrap.my_pkcs12() + # (a) rain check for sufficient config in sfi_config - # we don't allow to override these settings for now myslice_dict={} - myslice_keys=['backend', 'delegate', 'platform', 'username'] + myslice_keys=[ 'backend', 'delegate', 'platform', 'username'] for key in myslice_keys: - full_key="MYSLICE_" + key.upper() - value=getattr(self.config_instance,full_key,None) + value=None + # oct 2013 - I'm finding myself juggling with config files + # so a couple of command-line options can now override config + if hasattr(options,key) and getattr(options,key) is not None: + value=getattr(options,key) + else: + full_key="MYSLICE_" + key.upper() + value=getattr(self.config_instance,full_key,None) if value: myslice_dict[key]=value else: print "Unsufficient config, missing key %s in [myslice] section of sfi_config"%key if len(myslice_dict) != len(myslice_keys): sys.exit(1) # (b) figure whether we are PI for the authority where we belong - self.logger.info("Resolving our own id") + self.logger.info("Resolving our own id %s"%self.user) my_records=self.registry().Resolve(self.user,self.my_credential_string) if len(my_records)!=1: print "Cannot Resolve %s -- exiting"%self.user; sys.exit(1) my_record=my_records[0] @@ -1537,18 +1546,18 @@ $ sfi m my_auths = my_auths_all if options.delegate_auths: my_auths = list(set(my_auths_all).intersection(set(options.delegate_auths))) + self.logger.debug("Restricted to user-provided auths"%(my_auths)) - self.logger.info("Delegate PI creds for authorities: %s"%my_auths ) # (c) get the set of slices that we are in my_slices_all=my_record['reg-slices'] self.logger.info("Found %d slices that we are member of"%len(my_slices_all)) self.logger.debug("They are: %s"%(my_slices_all)) my_slices = my_slices_all + # if user provided slices, deal only with these - if they are found if options.delegate_slices: my_slices = list(set(my_slices_all).intersection(set(options.delegate_slices))) - - self.logger.info("Delegate slice creds for slices: %s"%my_slices) + self.logger.debug("Restricted to user-provided slices: %s"%(my_slices)) # (d) make sure we have *valid* credentials for all these hrn_credentials=[] @@ -1566,11 +1575,18 @@ $ sfi m hrn_delegated_credentials = [] for (hrn, htype, credential) in hrn_credentials: delegated_credential = self.client_bootstrap.delegate_credential_string (credential, delegatee_hrn, delegatee_type) - hrn_delegated_credentials.append ((hrn, htype, delegated_credential, )) + # save these so user can monitor what she's uploaded + filename = os.path.join ( self.options.sfi_dir, + "%s.%s_for_%s.%s.cred"%(hrn,htype,delegatee_hrn,delegatee_type)) + with file(filename,'w') as f: + f.write(delegated_credential) + self.logger.debug("(Over)wrote %s"%filename) + hrn_delegated_credentials.append ((hrn, htype, delegated_credential, filename, )) # (f) and finally upload them to manifold server # xxx todo add an option so the password can be set on the command line # (but *NOT* in the config file) so other apps can leverage this + self.logger.info("Uploading on backend at %s"%myslice_dict['backend']) uploader = ManifoldUploader (logger=self.logger, url=myslice_dict['backend'], platform=myslice_dict['platform'], @@ -1578,7 +1594,7 @@ $ sfi m password=options.password) uploader.prompt_all() (count_all,count_success)=(0,0) - for (hrn,htype,delegated_credential) in hrn_delegated_credentials: + for (hrn,htype,delegated_credential,filename) in hrn_delegated_credentials: # inspect inspect=Credential(string=delegated_credential) expire_datetime=inspect.get_expiration() @@ -1586,8 +1602,8 @@ $ sfi m if uploader.upload(delegated_credential,message=message): count_success+=1 count_all+=1 - self.logger.info("Successfully uploaded %d/%d credentials"%(count_success,count_all)) + # at first I thought we would want to save these, # like 'sfi delegate does' but on second thought # it is probably not helpful as people would not @@ -1595,17 +1611,18 @@ $ sfi m if count_success != count_all: sys.exit(1) return -# Thierry: I'm turning this off as a command, no idea what it's used for -# @register_command("cred","") + @register_command("cred","") def trusted(self, options, args): """ return the trusted certs at this interface (get_trusted_certs) """ trusted_certs = self.registry().get_trusted_certs() for trusted_cert in trusted_certs: + print "\n===========================================================\n" gid = GID(string=trusted_cert) gid.dump() cert = Certificate(string=trusted_cert) self.logger.debug('Sfi.trusted -> %r'%cert.get_subject()) + print "Certificate:\n%s\n\n"%trusted_cert return diff --git a/sfa/senslab/LDAPapi.py b/sfa/cortexlab/LDAPapi.py similarity index 51% rename from sfa/senslab/LDAPapi.py rename to sfa/cortexlab/LDAPapi.py index 6d34298c..15067ac5 100644 --- a/sfa/senslab/LDAPapi.py +++ b/sfa/cortexlab/LDAPapi.py @@ -1,167 +1,206 @@ +""" +This API is adapted for OpenLDAP. The file contains all LDAP classes and methods +needed to: +- Load the LDAP connection configuration file (login, address..) with LdapConfig +- Connect to LDAP with ldap_co +- Create a unique LDAP login and password for a user based on his email or last +name and first name with LoginPassword. +- Manage entries in LDAP using SFA records with LDAPapi (Search, Add, Delete, +Modify) + +""" import random from passlib.hash import ldap_salted_sha1 as lssha -from sfa.util.xrn import get_authority -import ldap -from sfa.util.config import Config +from sfa.util.xrn import get_authority +from sfa.util.sfalogging import logger +from sfa.util.config import Config +import ldap import ldap.modlist as modlist -from sfa.util.sfalogging import logger -import os.path -#API for OpenLDAP +import os.path class LdapConfig(): - def __init__(self, config_file = '/etc/sfa/ldap_config.py'): - + """ + Ldap configuration class loads the configuration file and sets the + ldap IP address, password, people dn, web dn, group dn. All these settings + were defined in a separate file ldap_config.py to avoid sharing them in + the SFA git as it contains sensible information. + + """ + def __init__(self, config_file='/etc/sfa/ldap_config.py'): + """Loads configuration from file /etc/sfa/ldap_config.py and set the + parameters for connection to LDAP. + + """ + try: execfile(config_file, self.__dict__) - + self.config_file = config_file # path to configuration data self.config_path = os.path.dirname(config_file) except IOError: raise IOError, "Could not find or load the configuration file: %s" \ - % config_file - - + % config_file + + class ldap_co: """ Set admin login and server configuration variables.""" - + def __init__(self): - #Senslab PROD LDAP parameters + """Fetch LdapConfig attributes (Ldap server connection parameters and + defines port , version and subtree scope. + + """ + #Iotlab PROD LDAP parameters self.ldapserv = None ldap_config = LdapConfig() self.config = ldap_config - self.ldapHost = ldap_config.LDAP_IP_ADDRESS + self.ldapHost = ldap_config.LDAP_IP_ADDRESS self.ldapPeopleDN = ldap_config.LDAP_PEOPLE_DN self.ldapGroupDN = ldap_config.LDAP_GROUP_DN self.ldapAdminDN = ldap_config.LDAP_WEB_DN self.ldapAdminPassword = ldap_config.LDAP_WEB_PASSWORD - - self.ldapPort = ldap.PORT - self.ldapVersion = ldap.VERSION3 + self.ldapVersion = ldap.VERSION3 self.ldapSearchScope = ldap.SCOPE_SUBTREE + def connect(self, bind=True): + """Enables connection to the LDAP server. + + :param bind: Set the bind parameter to True if a bind is needed + (for add/modify/delete operations). Set to False otherwise. + :type bind: boolean + :returns: dictionary with status of the connection. True if Successful, + False if not and in this case the error + message( {'bool', 'message'} ). + :rtype: dict - def connect(self, bind = True): - """ - Enables connection to the LDAP server. - :param bind : Set the bind parameter to True if a bind is needed - (for add/modify/delete operations). - Set to False otherwise. - :type bind : boolean - :return: dictionary with status of the connection. True if Successful, - False if not and in this case the error message( {'bool', 'message'} ) - :rtype:dict """ try: self.ldapserv = ldap.open(self.ldapHost) except ldap.LDAPError, error: - return {'bool' : False, 'message' : error } - + return {'bool': False, 'message': error} + # Bind with authentification - if(bind): + if(bind): return self.bind() - - else: - return {'bool': True} - + + else: + return {'bool': True} + def bind(self): - """ Binding method. - :return: dictionary with the bind status. True if Successful, - False if not and in this case the error message( {'bool', 'message'} ) + """ Binding method. + + :returns: dictionary with the bind status. True if Successful, + False if not and in this case the error message({'bool','message'}) :rtype: dict - + """ try: # Opens a connection after a call to ldap.open in connect: self.ldapserv = ldap.initialize("ldap://" + self.ldapHost) - - # Bind/authenticate with a user with apropriate + + # Bind/authenticate with a user with apropriate #rights to add objects - self.ldapserv.simple_bind_s(self.ldapAdminDN, \ - self.ldapAdminPassword) + self.ldapserv.simple_bind_s(self.ldapAdminDN, + self.ldapAdminPassword) except ldap.LDAPError, error: - return {'bool' : False, 'message' : error } + return {'bool': False, 'message': error} return {'bool': True} - + def close(self): - """ Close the LDAP connection. + """Close the LDAP connection. + Can throw an exception if the unbinding fails. + + :returns: dictionary with the bind status if the unbinding failed and + in this case the dict contains an error message. The dictionary keys + are : ({'bool','message'}) + :rtype: dict or None + """ try: self.ldapserv.unbind_s() except ldap.LDAPError, error: - return {'bool' : False, 'message' : error } - + return {'bool': False, 'message': error} + + class LoginPassword(): """ + Class to handle login and password generation, using custom login generation algorithm. + """ def __init__(self): """ - Sets password and login maximum length, and defines the characters - that can be found in a random generated password. + + Sets password and login maximum length, and defines the characters that + can be found in a random generated password. + """ - self.login_max_length = 8 + self.login_max_length = 8 self.length_password = 8 - self.chars_password = [ '!', '$', '(',')', '*', '+', ',', '-', '.', \ - '0', '1', '2', '3', '4', '5', '6', '7', '8', \ - '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', \ - 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', \ - 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', \ - '_', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', \ - 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p' ,'q', \ - 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', \ - '\''] - - - - + self.chars_password = ['!', '$', '(',')', '*', '+', ',', '-', '.', + '0', '1', '2', '3', '4', '5', '6', '7', '8', + '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', + 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + '_', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', + 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', + 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '\''] + @staticmethod def clean_user_names(record): """ - Removes special characters such as - '-', '_' , '[', ']' and ' ' from the first name and last name. + + Removes special characters such as '-', '_' , '[', ']' and ' ' from the + first name and last name. + :param record: user's record - :type record:dict - :return: lower_first_name and lower_last_name if they were found - in the user's record. Return None, none otherwise. + :type record: dict + :returns: lower_first_name and lower_last_name if they were found + in the user's record. Return None, none otherwise. :rtype: string, string or None, None. + """ if 'first_name' in record and 'last_name' in record: #Remove all special characters from first_name/last name - lower_first_name = record['first_name'].replace('-','')\ - .replace('_','').replace('[','')\ - .replace(']','').replace(' ','')\ - .lower() - lower_last_name = record['last_name'].replace('-','')\ - .replace('_','').replace('[','')\ - .replace(']','').replace(' ','')\ - .lower() + lower_first_name = record['first_name'].replace('-', '')\ + .replace('_', '').replace('[', '')\ + .replace(']', '').replace(' ', '')\ + .lower() + lower_last_name = record['last_name'].replace('-', '')\ + .replace('_', '').replace('[', '')\ + .replace(']', '').replace(' ', '')\ + .lower() return lower_first_name, lower_last_name else: return None, None - + @staticmethod def extract_name_from_email(record): """ - When there is no valid first name and last name in the record, + + When there is no valid first name and last name in the record, the email is used to generate the login. Here, we assume the email is firstname.lastname@something.smthg. The first name and last names are extracted from the email, special charcaters are removed and they are changed into lower case. + :param record: user's data - :type record:dict - :return: the first name and last name taken from the user's email. - lower_first_name, lower_last_name. + :type record: dict + :returns: the first name and last name taken from the user's email. + lower_first_name, lower_last_name. :rtype: string, string + """ email = record['email'] @@ -177,166 +216,177 @@ class LoginPassword(): lower_first_name = mail[0] lower_last_name = mail[1] break - - #Otherwise just take the part before the @ as the + + #Otherwise just take the part before the @ as the #lower_first_name and lower_last_name if lower_first_name is None: lower_first_name = email - lower_last_name = email - + lower_last_name = email + return lower_first_name, lower_last_name def get_user_firstname_lastname(self, record): - """Get the user first name and last name from the information - we have in the record. + """ + + Get the user first name and last name from the information we have in + the record. + :param record: user's information :type record: dict - :return: the user's first name and last name. - ..seealso: clean_user_names - ..seealso: extract_name_from_email + :returns: the user's first name and last name. + + .. seealso:: clean_user_names + .. seealso:: extract_name_from_email + """ lower_first_name, lower_last_name = self.clean_user_names(record) - - #No first name and last name check email + + #No first name and last name check email if lower_first_name is None and lower_last_name is None: lower_first_name, lower_last_name = \ - self.extract_name_from_email(record) - + self.extract_name_from_email(record) + return lower_first_name, lower_last_name - - + def choose_sets_chars_for_login(self, lower_first_name, lower_last_name): """ - Algorithm to select sets of characters from the first name and - last name, depending on the lenght of the last name and the - maximum login length which in our case is set to 8 charachetrs. + + Algorithm to select sets of characters from the first name and last + name, depending on the lenght of the last name and the maximum login + length which in our case is set to 8 characters. + :param lower_first_name: user's first name in lower case. :param lower_last_name: usr's last name in lower case. - :return: user's login - :rtype:string + :returns: user's login + :rtype: string + """ - length_last_name = len(lower_last_name) + length_last_name = len(lower_last_name) self.login_max_length = 8 - + #Try generating a unique login based on first name and last name - - if length_last_name >= self.login_max_length : + + if length_last_name >= self.login_max_length: login = lower_last_name[0:self.login_max_length] index = 0 - logger.debug("login : %s index : %s" %(login, index)) - elif length_last_name >= 4 : + logger.debug("login : %s index : %s" % (login, index)) + elif length_last_name >= 4: login = lower_last_name index = 0 - logger.debug("login : %s index : %s" %(login, index)) - elif length_last_name == 3 : + logger.debug("login : %s index : %s" % (login, index)) + elif length_last_name == 3: login = lower_first_name[0:1] + lower_last_name index = 1 - logger.debug("login : %s index : %s" %(login, index)) + logger.debug("login : %s index : %s" % (login, index)) elif length_last_name == 2: - if len ( lower_first_name) >=2: + if len(lower_first_name) >= 2: login = lower_first_name[0:2] + lower_last_name index = 2 - logger.debug("login : %s index : %s" %(login, index)) + logger.debug("login : %s index : %s" % (login, index)) else: logger.error("LoginException : \ Generation login error with \ minimum four characters") - else : + else: logger.error("LDAP LdapGenerateUniqueLogin failed : \ - impossible to generate unique login for %s %s" \ - %(lower_first_name,lower_last_name)) + impossible to generate unique login for %s %s" + % (lower_first_name, lower_last_name)) return index, login - - def generate_password(self): - - """Generate a password upon adding a new user in LDAP Directory - (8 characters length). The generated password is composed of characters - from the charsPassword list - :return: the randomly generated password + """ + + Generate a password upon adding a new user in LDAP Directory + (8 characters length). The generated password is composed of characters + from the chars_password list. + + :returns: the randomly generated password :rtype: string - + """ password = str() - + length = len(self.chars_password) for index in range(self.length_password): - char_index = random.randint(0, length-1) + char_index = random.randint(0, length - 1) password += self.chars_password[char_index] return password - + @staticmethod def encrypt_password(password): - """ Use passlib library to make a RFC2307 LDAP encrypted password - salt size = 8, use sha-1 algorithm. + """ + + Use passlib library to make a RFC2307 LDAP encrypted password salt size + is 8, use sha-1 algorithm. + :param password: password not encrypted. :type password: string - :return: Returns encrypted password. - :rtype:string + :returns: Returns encrypted password. + :rtype: string + """ - #Keep consistency with Java Senslab's LDAP API + #Keep consistency with Java Iotlab's LDAP API #RFC2307SSHAPasswordEncryptor so set the salt size to 8 bytes - return lssha.encrypt(password, salt_size = 8) - - - -class LDAPapi : + return lssha.encrypt(password, salt_size=8) + + +class LDAPapi: + """Defines functions to insert and search entries in the LDAP. + + .. note:: class supposes the unix schema is used + + """ def __init__(self): - logger.setLevelDebug() - + logger.setLevelDebug() + #SFA related config config = Config() - self.login_pwd = LoginPassword() + self.login_pwd = LoginPassword() self.authname = config.SFA_REGISTRY_ROOT_AUTH - - self.conn = ldap_co() - self.ldapUserQuotaNFS = self.conn.config.LDAP_USER_QUOTA_NFS - self.ldapUserUidNumberMin = self.conn.config.LDAP_USER_UID_NUMBER_MIN - self.ldapUserGidNumber = self.conn.config.LDAP_USER_GID_NUMBER - self.ldapUserHomePath = self.conn.config.LDAP_USER_HOME_PATH - + self.conn = ldap_co() + self.ldapUserQuotaNFS = self.conn.config.LDAP_USER_QUOTA_NFS + self.ldapUserUidNumberMin = self.conn.config.LDAP_USER_UID_NUMBER_MIN + self.ldapUserGidNumber = self.conn.config.LDAP_USER_GID_NUMBER + self.ldapUserHomePath = self.conn.config.LDAP_USER_HOME_PATH self.baseDN = self.conn.ldapPeopleDN + self.ldapShell = '/bin/bash' - - self.ldapShell = '/bin/bash' - - - def LdapGenerateUniqueLogin(self, record): """ - Generate login for adding a new user in LDAP Directory + + Generate login for adding a new user in LDAP Directory (four characters minimum length). Get proper last name and - first name so that the user's login can be generated. + first name so that the user's login can be generated. + :param record: Record must contain first_name and last_name. - :param record: dict - :return: the generated login for the user described with record if the - login generation is successful, None if it fails. + :type record: dict + :returns: the generated login for the user described with record if the + login generation is successful, None if it fails. :rtype: string or None - """ + + """ #For compatibility with other ldap func if 'mail' in record and 'email' not in record: record['email'] = record['mail'] - + lower_first_name, lower_last_name = \ - self.login_pwd.get_user_firstname_lastname(record) - - - index, login = self.login_pwd.choose_sets_chars_for_login( \ - lower_first_name, \ - lower_last_name) + self.login_pwd.get_user_firstname_lastname(record) + + index, login = self.login_pwd.choose_sets_chars_for_login( + lower_first_name, lower_last_name) + login_filter = '(uid=' + login + ')' get_attrs = ['uid'] - try : + try: #Check if login already in use - - while (len(self.LdapSearch(login_filter, get_attrs)) is not 0 ): - + + while (len(self.LdapSearch(login_filter, get_attrs)) is not 0): + index += 1 if index >= 9: logger.error("LoginException : Generation login error \ @@ -344,144 +394,145 @@ class LDAPapi : else: try: login = \ - lower_first_name[0:index] + \ - lower_last_name[0:self.login_pwd.login_max_length-index] - login_filter = '(uid='+ login+ ')' + lower_first_name[0:index] + \ + lower_last_name[0: + self.login_pwd.login_max_length + - index] + login_filter = '(uid=' + login + ')' except KeyError: print "lower_first_name - lower_last_name too short" - - logger.debug("LDAP.API \t LdapGenerateUniqueLogin login %s"%(login)) + + logger.debug("LDAP.API \t LdapGenerateUniqueLogin login %s" + % (login)) return login - - except ldap.LDAPError, error : - logger.log_exc("LDAP LdapGenerateUniqueLogin Error %s" %error) - return None + except ldap.LDAPError, error: + logger.log_exc("LDAP LdapGenerateUniqueLogin Error %s" % (error)) + return None def find_max_uidNumber(self): - - """Find the LDAP max uidNumber (POSIX uid attribute) . - Used when adding a new user in LDAP Directory - :return: max uidNumber + 1 - :rtype:string + """Find the LDAP max uidNumber (POSIX uid attribute). + + Used when adding a new user in LDAP Directory + + :returns: max uidNumber + 1 + :rtype: string + """ #First, get all the users in the LDAP get_attrs = "(uidNumber=*)" login_filter = ['uidNumber'] - result_data = self.LdapSearch(get_attrs, login_filter) + result_data = self.LdapSearch(get_attrs, login_filter) #It there is no user in LDAP yet, First LDAP user if result_data == []: max_uidnumber = self.ldapUserUidNumberMin #Otherwise, get the highest uidNumber else: - - uidNumberList = [int(r[1]['uidNumber'][0])for r in result_data ] + uidNumberList = [int(r[1]['uidNumber'][0])for r in result_data] logger.debug("LDAPapi.py \tfind_max_uidNumber \ - uidNumberList %s " %(uidNumberList)) + uidNumberList %s " % (uidNumberList)) max_uidnumber = max(uidNumberList) + 1 - + return str(max_uidnumber) - - + + def get_ssh_pkey(self, record): - """TODO ; Get ssh public key from sfa record + """TODO ; Get ssh public key from sfa record To be filled by N. Turro ? or using GID pl way? - + """ return 'A REMPLIR ' - + @staticmethod - #TODO Handle OR filtering in the ldap query when + #TODO Handle OR filtering in the ldap query when #dealing with a list of records instead of doing a for loop in GetPersons - def make_ldap_filters_from_record( record=None): - """ - Helper function to make LDAP filter requests out of SFA records. + def make_ldap_filters_from_record(record=None): + """Helper function to make LDAP filter requests out of SFA records. + :param record: user's sfa record. Should contain first_name,last_name, - email or mail, and if the record is enabled or not. If the dict - record does not have all of these, must at least contain the user's - email. + email or mail, and if the record is enabled or not. If the dict + record does not have all of these, must at least contain the user's + email. :type record: dict - :return: LDAP request + :returns: LDAP request :rtype: string + """ req_ldap = '' req_ldapdict = {} if record : - if 'first_name' in record and 'last_name' in record: - req_ldapdict['cn'] = str(record['first_name'])+" "\ - + str(record['last_name']) - if 'email' in record : + if 'first_name' in record and 'last_name' in record: + if record['first_name'] != record['last_name']: + req_ldapdict['cn'] = str(record['first_name'])+" "\ + + str(record['last_name']) + if 'email' in record: req_ldapdict['mail'] = record['email'] if 'mail' in record: req_ldapdict['mail'] = record['mail'] if 'enabled' in record: - if record['enabled'] == True : + if record['enabled'] is True: req_ldapdict['shadowExpire'] = '-1' else: req_ldapdict['shadowExpire'] = '0' - - #Hrn should not be part of the filter because the hrn - #presented by a certificate of a SFA user not imported in - #Senslab does not include the senslab login in it - #Plus, the SFA user may already have an account with senslab + + #Hrn should not be part of the filter because the hrn + #presented by a certificate of a SFA user not imported in + #Iotlab does not include the iotlab login in it + #Plus, the SFA user may already have an account with iotlab #using another login. - - logger.debug("\r\n \t LDAP.PY make_ldap_filters_from_record \ - record %s req_ldapdict %s" \ - %(record, req_ldapdict)) - + record %s req_ldapdict %s" + % (record, req_ldapdict)) + for k in req_ldapdict: - req_ldap += '('+ str(k)+ '=' + str(req_ldapdict[k]) + ')' - if len(req_ldapdict.keys()) >1 : + req_ldap += '(' + str(k) + '=' + str(req_ldapdict[k]) + ')' + if len(req_ldapdict.keys()) >1 : req_ldap = req_ldap[:0]+"(&"+req_ldap[0:] size = len(req_ldap) - req_ldap = req_ldap[:(size-1)] +')'+ req_ldap[(size-1):] + req_ldap = req_ldap[:(size-1)] + ')' + req_ldap[(size-1):] else: req_ldap = "(cn=*)" - + return req_ldap - + def make_ldap_attributes_from_record(self, record): - """When adding a new user to Senslab's LDAP, creates an attributes - dictionnary from the SFA record understandable by LDAP. - Generates the user's LDAP login. - User is automatically validated (account enabled) and described - as a SFA USER FROM OUTSIDE SENSLAB'. + """ + + When adding a new user to Iotlab's LDAP, creates an attributes + dictionnary from the SFA record understandable by LDAP. Generates the + user's LDAP login.User is automatically validated (account enabled) + and described as a SFA USER FROM OUTSIDE IOTLAB. + :param record: must contain the following keys and values: - first_name, last_name, mail, pkey (ssh key). + first_name, last_name, mail, pkey (ssh key). :type record: dict - - :return: dictionary of attributes using LDAP data structure - model. + :returns: dictionary of attributes using LDAP data structure model. :rtype: dict - + """ attrs = {} - attrs['objectClass'] = ["top", "person", "inetOrgPerson", \ - "organizationalPerson", "posixAccount", \ - "shadowAccount", "systemQuotas", \ - "ldapPublicKey"] - - - attrs['uid'] = self.LdapGenerateUniqueLogin(record) + attrs['objectClass'] = ["top", "person", "inetOrgPerson", + "organizationalPerson", "posixAccount", + "shadowAccount", "systemQuotas", + "ldapPublicKey"] + + attrs['uid'] = self.LdapGenerateUniqueLogin(record) try: attrs['givenName'] = str(record['first_name']).lower().capitalize() attrs['sn'] = str(record['last_name']).lower().capitalize() attrs['cn'] = attrs['givenName'] + ' ' + attrs['sn'] attrs['gecos'] = attrs['givenName'] + ' ' + attrs['sn'] - - except KeyError: + + except KeyError: attrs['givenName'] = attrs['uid'] attrs['sn'] = attrs['uid'] attrs['cn'] = attrs['uid'] attrs['gecos'] = attrs['uid'] - - - attrs['quota'] = self.ldapUserQuotaNFS + + attrs['quota'] = self.ldapUserQuotaNFS attrs['homeDirectory'] = self.ldapUserHomePath + attrs['uid'] attrs['loginShell'] = self.ldapShell attrs['gidNumber'] = self.ldapUserGidNumber @@ -490,28 +541,28 @@ class LDAPapi : try: attrs['sshPublicKey'] = record['pkey'] except KeyError: - attrs['sshPublicKey'] = self.get_ssh_pkey(record) - + attrs['sshPublicKey'] = self.get_ssh_pkey(record) + - #Password is automatically generated because SFA user don't go - #through the Senslab website used to register new users, + #Password is automatically generated because SFA user don't go + #through the Iotlab website used to register new users, #There is no place in SFA where users can enter such information #yet. - #If the user wants to set his own password , he must go to the Senslab + #If the user wants to set his own password , he must go to the Iotlab #website. password = self.login_pwd.generate_password() attrs['userPassword'] = self.login_pwd.encrypt_password(password) - + #Account automatically validated (no mail request to admins) #Set to 0 to disable the account, -1 to enable it, attrs['shadowExpire'] = '-1' - #Motivation field in Senslab + #Motivation field in Iotlab attrs['description'] = 'SFA USER FROM OUTSIDE SENSLAB' attrs['ou'] = 'SFA' #Optional: organizational unit #No info about those here: - attrs['l'] = 'To be defined'#Optional: Locality. + attrs['l'] = 'To be defined'#Optional: Locality. attrs['st'] = 'To be defined' #Optional: state or province (country). return attrs @@ -519,208 +570,216 @@ class LDAPapi : def LdapAddUser(self, record) : - """Add SFA user to LDAP if it is not in LDAP yet. - :param record: dictionnary with the user's data. - - :return: a dictionary with the status (Fail= False, Success= True) - and the uid of the newly added user if successful, or the error - meassage it is not. Dict has keys bool and message in case of failure, - and bool uid in case of success. + """Add SFA user to LDAP if it is not in LDAP yet. + + :param record: dictionnary with the user's data. + :returns: a dictionary with the status (Fail= False, Success= True) + and the uid of the newly added user if successful, or the error + message it is not. Dict has keys bool and message in case of + failure, and bool uid in case of success. :rtype: dict - - ..seealso: make_ldap_filters_from_record - + + .. seealso:: make_ldap_filters_from_record + """ logger.debug(" \r\n \t LDAP LdapAddUser \r\n\r\n ================\r\n ") user_ldap_attrs = self.make_ldap_attributes_from_record(record) - #Check if user already in LDAP wih email, first name and last name filter_by = self.make_ldap_filters_from_record(user_ldap_attrs) user_exist = self.LdapSearch(filter_by) if user_exist: logger.warning(" \r\n \t LDAP LdapAddUser user %s %s \ - already exists" %(user_ldap_attrs['sn'], \ - user_ldap_attrs['mail'])) + already exists" % (user_ldap_attrs['sn'], + user_ldap_attrs['mail'])) return {'bool': False} - + #Bind to the server result = self.conn.connect() - + if(result['bool']): - + # A dict to help build the "body" of the object - - logger.debug(" \r\n \t LDAP LdapAddUser attrs %s " %user_ldap_attrs) + logger.debug(" \r\n \t LDAP LdapAddUser attrs %s " + % user_ldap_attrs) # The dn of our new entry/object - dn = 'uid=' + user_ldap_attrs['uid'] + "," + self.baseDN + dn = 'uid=' + user_ldap_attrs['uid'] + "," + self.baseDN try: ldif = modlist.addModlist(user_ldap_attrs) - logger.debug("LDAPapi.py add attrs %s \r\n ldif %s"\ - %(user_ldap_attrs, ldif) ) + logger.debug("LDAPapi.py add attrs %s \r\n ldif %s" + % (user_ldap_attrs, ldif)) self.conn.ldapserv.add_s(dn, ldif) - - logger.info("Adding user %s login %s in LDAP" \ - %(user_ldap_attrs['cn'] , user_ldap_attrs['uid'])) - - + + logger.info("Adding user %s login %s in LDAP" + % (user_ldap_attrs['cn'], user_ldap_attrs['uid'])) except ldap.LDAPError, error: - logger.log_exc("LDAP Add Error %s" %error) - return {'bool' : False, 'message' : error } - + logger.log_exc("LDAP Add Error %s" % error) + return {'bool': False, 'message': error} + self.conn.close() - return {'bool': True, 'uid':user_ldap_attrs['uid']} - else: + return {'bool': True, 'uid': user_ldap_attrs['uid']} + else: return result - def LdapDelete(self, person_dn): - """ - Deletes a person in LDAP. Uses the dn of the user. + """Deletes a person in LDAP. Uses the dn of the user. + :param person_dn: user's ldap dn. :type person_dn: string - :return: dictionary with bool True if successful, bool False - and the error if not. - :rtype:dict + :returns: dictionary with bool True if successful, bool False + and the error if not. + :rtype: dict + """ - #Connect and bind + #Connect and bind result = self.conn.connect() if(result['bool']): try: self.conn.ldapserv.delete_s(person_dn) self.conn.close() return {'bool': True} - + except ldap.LDAPError, error: - logger.log_exc("LDAP Delete Error %s" %error) + logger.log_exc("LDAP Delete Error %s" % error) return {'bool': False, 'message': error} - - - def LdapDeleteUser(self, record_filter): - """ - Deletes a SFA person in LDAP, based on the user's hrn. - :param record_filter: Filter to find the user to be deleted. Must - contain at least the user's email. + + def LdapDeleteUser(self, record_filter): + """Deletes a SFA person in LDAP, based on the user's hrn. + + :param record_filter: Filter to find the user to be deleted. Must + contain at least the user's email. :type record_filter: dict - :return: dict with bool True if successful, bool False and error message - otherwise - :rtype:dict - ..seealso: LdapFindUser docstring for more info on record filter. - ..seealso: LdapDelete for user deletion + :returns: dict with bool True if successful, bool False and error + message otherwise. + :rtype: dict + + .. seealso:: LdapFindUser docstring for more info on record filter. + .. seealso:: LdapDelete for user deletion + """ - #Find uid of the person + #Find uid of the person person = self.LdapFindUser(record_filter, []) - logger.debug("LDAPapi.py \t LdapDeleteUser record %s person %s" \ - %(record_filter, person)) + logger.debug("LDAPapi.py \t LdapDeleteUser record %s person %s" + % (record_filter, person)) if person: - dn = 'uid=' + person['uid'] + "," + self.baseDN + dn = 'uid=' + person['uid'] + "," + self.baseDN else: return {'bool': False} - + result = self.LdapDelete(dn) return result - - def LdapModify(self, dn, old_attributes_dict, new_attributes_dict): + def LdapModify(self, dn, old_attributes_dict, new_attributes_dict): """ Modifies a LDAP entry, replaces user's old attributes with the new ones given. - :param dn: user's absolute name in the LDAP hierarchy. + + :param dn: user's absolute name in the LDAP hierarchy. :param old_attributes_dict: old user's attributes. Keys must match - the ones used in the LDAP model. + the ones used in the LDAP model. :param new_attributes_dict: new user's attributes. Keys must match - the ones used in the LDAP model. + the ones used in the LDAP model. :type dn: string :type old_attributes_dict: dict :type new_attributes_dict: dict - :return: dict bool True if Successful, bool False if not. - :rtype:dict + :returns: dict bool True if Successful, bool False if not. + :rtype: dict + """ - + ldif = modlist.modifyModlist(old_attributes_dict, new_attributes_dict) - # Connect and bind/authenticate - result = self.conn.connect() - if (result['bool']): + # Connect and bind/authenticate + result = self.conn.connect() + if (result['bool']): try: self.conn.ldapserv.modify_s(dn, ldif) self.conn.close() - return {'bool' : True } + return {'bool': True} except ldap.LDAPError, error: - logger.log_exc("LDAP LdapModify Error %s" %error) - return {'bool' : False } - - + logger.log_exc("LDAP LdapModify Error %s" % error) + return {'bool': False} + + def LdapModifyUser(self, user_record, new_attributes_dict): """ - Gets the record from one user based on the user sfa record - and changes the attributes according to the specified new_attributes. - Do not use this if we need to modify the uid. Use a ModRDN - #operation instead ( modify relative DN ) + + Gets the record from one user based on the user sfa recordand changes + the attributes according to the specified new_attributes. Do not use + this if we need to modify the uid. Use a ModRDN operation instead + ( modify relative DN ). + :param user_record: sfa user record. :param new_attributes_dict: new user attributes, keys must be the - same as the LDAP model. + same as the LDAP model. :type user_record: dict :type new_attributes_dict: dict - :return: bool True if successful, bool False if not. + :returns: bool True if successful, bool False if not. :rtype: dict - ..seealso: make_ldap_filters_from_record for info on what is mandatory - in the user_record. - ..seealso: make_ldap_attributes_from_record for the LDAP objectclass. + + .. seealso:: make_ldap_filters_from_record for info on what is mandatory + in the user_record. + .. seealso:: make_ldap_attributes_from_record for the LDAP objectclass. + """ if user_record is None: logger.error("LDAP \t LdapModifyUser Need user record ") - return {'bool': False} - - #Get all the attributes of the user_uid_login + return {'bool': False} + + #Get all the attributes of the user_uid_login #person = self.LdapFindUser(record_filter,[]) req_ldap = self.make_ldap_filters_from_record(user_record) person_list = self.LdapSearch(req_ldap, []) - logger.debug("LDAPapi.py \t LdapModifyUser person_list : %s" \ - %(person_list)) - if person_list and len(person_list) > 1 : + logger.debug("LDAPapi.py \t LdapModifyUser person_list : %s" + % (person_list)) + + if person_list and len(person_list) > 1: logger.error("LDAP \t LdapModifyUser Too many users returned") return {'bool': False} - if person_list is None : - logger.error("LDAP \t LdapModifyUser User %s doesn't exist "\ - %(user_record)) - return {'bool': False} - + if person_list is None: + logger.error("LDAP \t LdapModifyUser User %s doesn't exist " + % (user_record)) + return {'bool': False} + # The dn of our existing entry/object #One result only from ldapSearch person = person_list[0][1] - dn = 'uid=' + person['uid'][0] + "," + self.baseDN - + dn = 'uid=' + person['uid'][0] + "," + self.baseDN + if new_attributes_dict: old = {} for k in new_attributes_dict: if k not in person: - old[k] = '' - else : + old[k] = '' + else: old[k] = person[k] - logger.debug(" LDAPapi.py \t LdapModifyUser new_attributes %s"\ - %( new_attributes_dict)) + logger.debug(" LDAPapi.py \t LdapModifyUser new_attributes %s" + % (new_attributes_dict)) result = self.LdapModify(dn, old, new_attributes_dict) return result else: logger.error("LDAP \t LdapModifyUser No new attributes given. ") - return {'bool': False} - - - - - def LdapMarkUserAsDeleted(self, record): - """ - Sets shadowExpire to 0, disabling the user in LDAP. - Calls LdapModifyUser to change the shadowExpire of the user. + return {'bool': False} + + + def LdapMarkUserAsDeleted(self, record): + """ + + Sets shadowExpire to 0, disabling the user in LDAP. Calls LdapModifyUser + to change the shadowExpire of the user. + :param record: the record of the user who has to be disabled. + Should contain first_name,last_name, email or mail, and if the + record is enabled or not. If the dict record does not have all of + these, must at least contain the user's email. :type record: dict - :return: bool True if successful or bool False if not - :rtype:dict - ..seealso: LdapModifyUser + :returns: {bool: True} if successful or {bool: False} if not + :rtype: dict + + .. seealso:: LdapModifyUser, make_ldap_attributes_from_record """ - + new_attrs = {} #Disable account new_attrs['shadowExpire'] = '0' @@ -728,217 +787,238 @@ class LDAPapi : ret = self.LdapModifyUser(record, new_attrs) return ret - def LdapResetPassword(self, record): - """ - Resets password for the user whose record is the parameter and changes - the corresponding entry in the LDAP. - + """Resets password for the user whose record is the parameter and + changes the corresponding entry in the LDAP. + + :param record: user's sfa record whose Ldap password must be reset. + Should contain first_name,last_name, + email or mail, and if the record is enabled or not. If the dict + record does not have all of these, must at least contain the user's + email. + :type record: dict + :returns: return value of LdapModifyUser. True if successful, False + otherwise. + + .. seealso:: LdapModifyUser, make_ldap_attributes_from_record + """ password = self.login_pwd.generate_password() attrs = {} attrs['userPassword'] = self.login_pwd.encrypt_password(password) - logger.debug("LDAP LdapResetPassword encrypt_password %s"\ - %(attrs['userPassword'])) + logger.debug("LDAP LdapResetPassword encrypt_password %s" + % (attrs['userPassword'])) result = self.LdapModifyUser(record, attrs) return result - - - def LdapSearch (self, req_ldap = None, expected_fields = None ): - """ - Used to search directly in LDAP, by using ldap filters and - return fields. - When req_ldap is None, returns all the entries in the LDAP. - - """ - result = self.conn.connect(bind = False) - if (result['bool']) : - + + + def LdapSearch(self, req_ldap=None, expected_fields=None): + """ + Used to search directly in LDAP, by using ldap filters and return + fields. When req_ldap is None, returns all the entries in the LDAP. + + :param req_ldap: ldap style request, with appropriate filters, + example: (cn=*). + :param expected_fields: Fields in the user ldap entry that has to be + returned. If None is provided, will return 'mail', 'givenName', + 'sn', 'uid', 'sshPublicKey', 'shadowExpire'. + :type req_ldap: string + :type expected_fields: list + + .. seealso:: make_ldap_filters_from_record for req_ldap format. + + """ + result = self.conn.connect(bind=False) + if (result['bool']): + return_fields_list = [] - if expected_fields == None : - return_fields_list = ['mail', 'givenName', 'sn', 'uid', \ - 'sshPublicKey', 'shadowExpire'] - else : + if expected_fields is None: + return_fields_list = ['mail', 'givenName', 'sn', 'uid', + 'sshPublicKey', 'shadowExpire'] + else: return_fields_list = expected_fields - #No specifc request specified, get the whole LDAP - if req_ldap == None: + #No specifc request specified, get the whole LDAP + if req_ldap is None: req_ldap = '(cn=*)' - + logger.debug("LDAP.PY \t LdapSearch req_ldap %s \ return_fields_list %s" \ %(req_ldap, return_fields_list)) try: msg_id = self.conn.ldapserv.search( - self.baseDN,ldap.SCOPE_SUBTREE,\ - req_ldap, return_fields_list) - #Get all the results matching the search from ldap in one + self.baseDN, ldap.SCOPE_SUBTREE, + req_ldap, return_fields_list) + #Get all the results matching the search from ldap in one #shot (1 value) result_type, result_data = \ - self.conn.ldapserv.result(msg_id, 1) + self.conn.ldapserv.result(msg_id, 1) self.conn.close() - logger.debug("LDAP.PY \t LdapSearch result_data %s"\ - %(result_data)) + logger.debug("LDAP.PY \t LdapSearch result_data %s" + % (result_data)) return result_data - - except ldap.LDAPError, error : - logger.log_exc("LDAP LdapSearch Error %s" %error) + + except ldap.LDAPError, error: + logger.log_exc("LDAP LdapSearch Error %s" % error) return [] - + else: - logger.error("LDAP.PY \t Connection Failed" ) - return - - + logger.error("LDAP.PY \t Connection Failed") + return + def _process_ldap_info_for_all_users(self, result_data): - """ - Process the data of all enabled users in LDAP. + """Process the data of all enabled users in LDAP. + :param result_data: Contains information of all enabled users in LDAP - and is coming from LdapSearch. + and is coming from LdapSearch. :param result_data: list - ..seealso: LdapSearch + + .. seealso:: LdapSearch + """ results = [] + logger.debug(" LDAP.py _process_ldap_info_for_all_users result_data %s " + % (result_data)) for ldapentry in result_data: - logger.debug(" LDAP.py LdapFindUser ldapentry name : %s " \ - %(ldapentry[1]['uid'][0])) + logger.debug(" LDAP.py _process_ldap_info_for_all_users \ + ldapentry name : %s " % (ldapentry[1]['uid'][0])) tmpname = ldapentry[1]['uid'][0] hrn = self.authname + "." + tmpname - + tmpemail = ldapentry[1]['mail'][0] if ldapentry[1]['mail'][0] == "unknown": tmpemail = None - try: - results.append( { - 'type': 'user', - 'pkey': ldapentry[1]['sshPublicKey'][0], - #'uid': ldapentry[1]['uid'][0], - 'uid': tmpname , - 'email':tmpemail, - #'email': ldapentry[1]['mail'][0], - 'first_name': ldapentry[1]['givenName'][0], - 'last_name': ldapentry[1]['sn'][0], - #'phone': 'none', - 'serial': 'none', - 'authority': self.authname, - 'peer_authority': '', - 'pointer' : -1, - 'hrn': hrn, - } ) + results.append({ + 'type': 'user', + 'pkey': ldapentry[1]['sshPublicKey'][0], + #'uid': ldapentry[1]['uid'][0], + 'uid': tmpname , + 'email':tmpemail, + #'email': ldapentry[1]['mail'][0], + 'first_name': ldapentry[1]['givenName'][0], + 'last_name': ldapentry[1]['sn'][0], + #'phone': 'none', + 'serial': 'none', + 'authority': self.authname, + 'peer_authority': '', + 'pointer': -1, + 'hrn': hrn, + }) except KeyError, error: - logger.log_exc("LDAPapi.PY \t LdapFindUser EXCEPTION %s" \ - %(error)) + logger.log_exc("LDAPapi.PY \t LdapFindUser EXCEPTION %s" + % (error)) return - - return results - + + return results + def _process_ldap_info_for_one_user(self, record, result_data): - """ + """ + Put the user's ldap data into shape. Only deals with one user - record and one user data from ldap. - :param record: user record + record and one user data from ldap. + + :param record: user record :param result_data: Raw ldap data coming from LdapSearch - :return: user's data dict with 'type','pkey','uid', 'email', - 'first_name' 'last_name''serial''authority''peer_authority' - 'pointer''hrn' + :returns: user's data dict with 'type','pkey','uid', 'email', + 'first_name' 'last_name''serial''authority''peer_authority' + 'pointer''hrn' :type record: dict :type result_data: list - :rtype :dict + :rtype :dict + """ #One entry only in the ldap data because we used a filter #to find one user only ldapentry = result_data[0][1] - logger.debug("LDAP.PY \t LdapFindUser ldapentry %s" %(ldapentry)) + logger.debug("LDAP.PY \t LdapFindUser ldapentry %s" % (ldapentry)) tmpname = ldapentry['uid'][0] tmpemail = ldapentry['mail'][0] if ldapentry['mail'][0] == "unknown": tmpemail = None - + parent_hrn = None - peer_authority = None + peer_authority = None if 'hrn' in record: hrn = record['hrn'] parent_hrn = get_authority(hrn) if parent_hrn != self.authname: peer_authority = parent_hrn - #In case the user was not imported from Senslab LDAP - #but from another federated site, has an account in - #senslab but currently using his hrn from federated site - #then the login is different from the one found in its hrn + #In case the user was not imported from Iotlab LDAP + #but from another federated site, has an account in + #iotlab but currently using his hrn from federated site + #then the login is different from the one found in its hrn if tmpname != hrn.split('.')[1]: hrn = None else: hrn = None - - - - results = { - 'type': 'user', - 'pkey': ldapentry['sshPublicKey'], - #'uid': ldapentry[1]['uid'][0], - 'uid': tmpname , - 'email':tmpemail, - #'email': ldapentry[1]['mail'][0], - 'first_name': ldapentry['givenName'][0], - 'last_name': ldapentry['sn'][0], - #'phone': 'none', - 'serial': 'none', - 'authority': parent_hrn, - 'peer_authority': peer_authority, - 'pointer' : -1, - 'hrn': hrn, + + results = { + 'type': 'user', + 'pkey': ldapentry['sshPublicKey'], + #'uid': ldapentry[1]['uid'][0], + 'uid': tmpname, + 'email': tmpemail, + #'email': ldapentry[1]['mail'][0], + 'first_name': ldapentry['givenName'][0], + 'last_name': ldapentry['sn'][0], + #'phone': 'none', + 'serial': 'none', + 'authority': parent_hrn, + 'peer_authority': peer_authority, + 'pointer': -1, + 'hrn': hrn, } return results - - - def LdapFindUser(self, record = None, is_user_enabled=None, \ - expected_fields = None): + + def LdapFindUser(self, record=None, is_user_enabled=None, + expected_fields=None): """ - Search a SFA user with a hrn. User should be already registered - in Senslab LDAP. + + Search a SFA user with a hrn. User should be already registered + in Iotlab LDAP. + :param record: sfa user's record. Should contain first_name,last_name, - email or mail. If no record is provided, returns all the users found - in LDAP. + email or mail. If no record is provided, returns all the users found + in LDAP. :type record: dict - :param is_user_enabled: is the user's senslab account already valid. - :type is_user_enabled: Boolean. - :return: LDAP entries from ldap matching the filter provided. Returns - a single entry if one filter has been given and a list of - entries otherwise. + :param is_user_enabled: is the user's iotlab account already valid. + :type is_user_enabled: Boolean. + :returns: LDAP entries from ldap matching the filter provided. Returns + a single entry if one filter has been given and a list of + entries otherwise. :rtype: dict or list - """ + + """ custom_record = {} - if is_user_enabled: + if is_user_enabled: custom_record['enabled'] = is_user_enabled - if record: + if record: custom_record.update(record) - - req_ldap = self.make_ldap_filters_from_record(custom_record) + req_ldap = self.make_ldap_filters_from_record(custom_record) return_fields_list = [] - if expected_fields == None : - return_fields_list = ['mail', 'givenName', 'sn', 'uid', \ - 'sshPublicKey'] - else : + if expected_fields is None: + return_fields_list = ['mail', 'givenName', 'sn', 'uid', + 'sshPublicKey'] + else: return_fields_list = expected_fields - - result_data = self.LdapSearch(req_ldap, return_fields_list ) - logger.debug("LDAP.PY \t LdapFindUser result_data %s" %(result_data)) - - if len(result_data) is 0: + + result_data = self.LdapSearch(req_ldap, return_fields_list) + logger.debug("LDAP.PY \t LdapFindUser result_data %s" % (result_data)) + + if len(result_data) == 0: return None #Asked for a specific user - if record : + if record is not None: results = self._process_ldap_info_for_one_user(record, result_data) - + else: #Asked for all users in ldap results = self._process_ldap_info_for_all_users(result_data) - return results - + return results \ No newline at end of file diff --git a/sfa/senslab/__init__.py b/sfa/cortexlab/__init__.py similarity index 100% rename from sfa/senslab/__init__.py rename to sfa/cortexlab/__init__.py diff --git a/sfa/cortexlab/cortexlabaggregate.py b/sfa/cortexlab/cortexlabaggregate.py new file mode 100644 index 00000000..68cb1ec0 --- /dev/null +++ b/sfa/cortexlab/cortexlabaggregate.py @@ -0,0 +1,462 @@ +""" +File providing methods to generate valid RSpecs for the Iotlab testbed. +Contains methods to get information on slice, slivers, nodes and leases, +formatting them and turn it into a RSpec. +""" +from sfa.util.xrn import hrn_to_urn, urn_to_hrn, get_authority + +from sfa.rspecs.rspec import RSpec +#from sfa.rspecs.elements.location import Location +from sfa.rspecs.elements.hardware_type import HardwareType +from sfa.rspecs.elements.login import Login +from sfa.rspecs.elements.services import Services +from sfa.rspecs.elements.sliver import Sliver +from sfa.rspecs.elements.lease import Lease +from sfa.rspecs.elements.granularity import Granularity +from sfa.rspecs.version_manager import VersionManager + +from sfa.rspecs.elements.versions.cortexlabv1Node import IotlabPosition, \ + IotlabNode, IotlabLocation, IotlabMobility + +from sfa.util.sfalogging import logger +from sfa.util.xrn import Xrn + + +def cortexlab_xrn_to_hostname(xrn): + """Returns a node's hostname from its xrn. + :param xrn: The nodes xrn identifier. + :type xrn: Xrn (from sfa.util.xrn) + + :returns: node's hostname. + :rtype: string + + """ + return Xrn.unescape(Xrn(xrn=xrn, type='node').get_leaf()) + + +def cortexlab_xrn_object(root_auth, hostname): + """Creates a valid xrn object from the node's hostname and the authority + of the SFA server. + + :param hostname: the node's hostname. + :param root_auth: the SFA root authority. + :type hostname: string + :type root_auth: string + + :returns: the cortexlab node's xrn + :rtype: Xrn + + """ + return Xrn('.'.join([root_auth, Xrn.escape(hostname)]), type='node') + + +class CortexlabAggregate: + """Aggregate manager class for Iotlab. """ + + sites = {} + nodes = {} + api = None + interfaces = {} + links = {} + node_tags = {} + + prepared = False + + user_options = {} + + def __init__(self, driver): + self.driver = driver + + def get_slice_and_slivers(self, slice_xrn, login=None): + """ + Get the slices and the associated leases if any, from the cortexlab + testbed. One slice can have mutliple leases. + For each slice, get the nodes in the associated lease + and create a sliver with the necessary info and insert it into the + sliver dictionary, keyed on the node hostnames. + Returns a dict of slivers based on the sliver's node_id. + Called by get_rspec. + + + :param slice_xrn: xrn of the slice + :param login: user's login on cortexlab ldap + + :type slice_xrn: string + :type login: string + :returns: a list of slices dict and a list of Sliver object + :rtype: (list, list) + + .. note: There is no real slivers in cortexlab, only leases. The goal + is to be consistent with the SFA standard. + + """ + slivers = {} + sfa_slice = None + if slice_xrn is None: + return (sfa_slice, slivers) + slice_urn = hrn_to_urn(slice_xrn, 'slice') + slice_hrn, _ = urn_to_hrn(slice_xrn) + slice_name = slice_hrn + + slices = self.driver.cortexlab_api.GetSlices(slice_filter=str(slice_name), + slice_filter_type='slice_hrn', + login=login) + + logger.debug("CortexlabAggregate api \tget_slice_and_slivers \ + slice_hrn %s \r\n slices %s self.driver.hrn %s" + % (slice_hrn, slices, self.driver.hrn)) + + if slices == []: + return (sfa_slice, slivers) + + # sort slivers by node id , if there is a job + #and therefore, node allocated to this slice + for sfa_slice in slices: + try: + node_ids_list = sfa_slice['node_ids'] + except KeyError: + logger.log_exc("CortexlabAggregate \t \ + get_slice_and_slivers No nodes in the slice \ + - KeyError ") + node_ids_list = [] + continue + + for node in node_ids_list: + sliver_xrn = Xrn(slice_urn, type='sliver', id=node) + sliver_xrn.set_authority(self.driver.hrn) + sliver = Sliver({'sliver_id': sliver_xrn.urn, + 'name': sfa_slice['hrn'], + 'type': 'cortexlab-node', + 'tags': []}) + + slivers[node] = sliver + + #Add default sliver attribute : + #connection information for cortexlab + if get_authority(sfa_slice['hrn']) == self.driver.cortexlab_api.root_auth: + tmp = sfa_slice['hrn'].split('.') + ldap_username = tmp[1].split('_')[0] + ssh_access = None + slivers['default_sliver'] = {'ssh': ssh_access, + 'login': ldap_username} + + #TODO get_slice_and_slivers Find the login of the external user + + logger.debug("CortexlabAggregate api get_slice_and_slivers slivers %s " + % (slivers)) + return (slices, slivers) + + + def get_nodes(self, slices=None, slivers=[], options=None): + """Returns the nodes in the slice using the rspec format, with all the + nodes' properties. + + Fetch the nodes ids in the slices dictionary and get all the nodes + properties from OAR. Makes a rspec dicitonary out of this and returns + it. If the slice does not have any job running or scheduled, that is + it has no reserved nodes, then returns an empty list. + + :param slices: list of slices (record dictionaries) + :param slivers: the list of slivers in all the slices + :type slices: list of dicts + :type slivers: list of Sliver object (dictionaries) + :returns: An empty list if the slice has no reserved nodes, a rspec + list with all the nodes and their properties (a dict per node) + otherwise. + :rtype: list + + .. seealso:: get_slice_and_slivers + + """ + # NT: the semantic of this function is not clear to me : + # if slice is not defined, then all the nodes should be returned + # if slice is defined, we should return only the nodes that + # are part of this slice + # but what is the role of the slivers parameter ? + # So i assume that slice['node_ids'] will be the same as slivers for us + slice_nodes_list = [] + if slices is not None: + for one_slice in slices: + try: + slice_nodes_list = one_slice['node_ids'] + # if we are dealing with a slice that has no node just + # return an empty list. In cortexlab a slice can have multiple + # jobs scheduled, so it either has at least one lease or + # not at all. + except KeyError: + return [] + + # get the granularity in second for the reservation system + grain = self.driver.cortexlab_api.GetLeaseGranularity() + + nodes = self.driver.cortexlab_api.GetNodes() + + nodes_dict = {} + + #if slices, this means we got to list all the nodes given to this slice + # Make a list of all the nodes in the slice before getting their + #attributes + rspec_nodes = [] + + logger.debug("CortexlabAggregate api get_nodes slices %s " + % (slices)) + + reserved_nodes = self.driver.cortexlab_api.GetNodesCurrentlyInUse() + logger.debug("CortexlabAggregate api get_nodes slice_nodes_list %s " + % (slice_nodes_list)) + for node in nodes: + nodes_dict[node['node_id']] = node + if slice_nodes_list == [] or node['hostname'] in slice_nodes_list: + + rspec_node = IotlabNode() + # xxx how to retrieve site['login_base'] + #site_id=node['site_id'] + #site=sites_dict[site_id] + + # rspec_node['mobile'] = node['mobile'] + rspec_node['archi'] = node['archi'] + rspec_node['radio'] = node['radio'] + + cortexlab_xrn = cortexlab_xrn_object(self.driver.cortexlab_api.root_auth, + node['hostname']) + rspec_node['component_id'] = cortexlab_xrn.urn + rspec_node['component_name'] = node['hostname'] + rspec_node['component_manager_id'] = \ + hrn_to_urn(self.driver.cortexlab_api.root_auth, + 'authority+sa') + + # Iotlab's nodes are federated : there is only one authority + # for all Iotlab sites, registered in SFA. + # Removing the part including the site + # in authority_id SA 27/07/12 + rspec_node['authority_id'] = rspec_node['component_manager_id'] + + # do not include boot state ( element) + #in the manifest rspec + + rspec_node['boot_state'] = node['boot_state'] + if node['hostname'] in reserved_nodes: + rspec_node['boot_state'] = "Reserved" + rspec_node['exclusive'] = 'true' + rspec_node['hardware_types'] = [HardwareType({'name': + 'cortexlab-node'})] + + + location = IotlabLocation({'country':'France', 'site': + node['site']}) + rspec_node['location'] = location + + # Adding mobility of the node in the rspec + mobility = IotlabMobility() + for field in mobility: + try: + mobility[field] = node[field] + except KeyError, error: + logger.log_exc("CortexlabAggregate\t get_nodes \ + mobility %s " % (error)) + rspec_node['mobility'] = mobility + + position = IotlabPosition() + for field in position: + try: + position[field] = node[field] + except KeyError, error: + logger.log_exc("CortexlabAggregate\t get_nodes \ + position %s " % (error)) + + rspec_node['position'] = position + #rspec_node['interfaces'] = [] + + # Granularity + granularity = Granularity({'grain': grain}) + rspec_node['granularity'] = granularity + rspec_node['tags'] = [] + if node['hostname'] in slivers: + # add sliver info + sliver = slivers[node['hostname']] + rspec_node['sliver_id'] = sliver['sliver_id'] + rspec_node['client_id'] = node['hostname'] + rspec_node['slivers'] = [sliver] + + # slivers always provide the ssh service + login = Login({'authentication': 'ssh-keys', + 'hostname': node['hostname'], 'port': '22', + 'username': sliver['name']}) + service = Services({'login': login}) + rspec_node['services'] = [service] + rspec_nodes.append(rspec_node) + + return (rspec_nodes) + + def get_all_leases(self, ldap_username): + """ + + Get list of lease dictionaries which all have the mandatory keys + ('lease_id', 'hostname', 'site_id', 'name', 'start_time', 'duration'). + All the leases running or scheduled are returned. + + :param ldap_username: if ldap uid is not None, looks for the leases + belonging to this user. + :type ldap_username: string + :returns: rspec lease dictionary with keys lease_id, component_id, + slice_id, start_time, duration. + :rtype: dict + + .. note::There is no filtering of leases within a given time frame. + All the running or scheduled leases are returned. options + removed SA 15/05/2013 + + + """ + + #now = int(time.time()) + #lease_filter = {'clip': now } + + #if slice_record: + #lease_filter.update({'name': slice_record['name']}) + + #leases = self.driver.cortexlab_api.GetLeases(lease_filter) + + logger.debug("CortexlabAggregate get_all_leases ldap_username %s " + % (ldap_username)) + leases = self.driver.cortexlab_api.GetLeases(login=ldap_username) + grain = self.driver.cortexlab_api.GetLeaseGranularity() + # site_ids = [] + rspec_leases = [] + for lease in leases: + #as many leases as there are nodes in the job + for node in lease['reserved_nodes']: + rspec_lease = Lease() + rspec_lease['lease_id'] = lease['lease_id'] + #site = node['site_id'] + cortexlab_xrn = cortexlab_xrn_object(self.driver.cortexlab_api.root_auth, + node) + rspec_lease['component_id'] = cortexlab_xrn.urn + #rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn,\ + #site, node['hostname']) + try: + rspec_lease['slice_id'] = lease['slice_id'] + except KeyError: + #No info on the slice used in cortexlab_xp table + pass + rspec_lease['start_time'] = lease['t_from'] + rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) \ + / grain + rspec_leases.append(rspec_lease) + return rspec_leases + + def get_rspec(self, slice_xrn=None, login=None, version=None, + options=None): + """ + Returns xml rspec: + - a full advertisement rspec with the testbed resources if slice_xrn is + not specified.If a lease option is given, also returns the leases + scheduled on the testbed. + - a manifest Rspec with the leases and nodes in slice's leases if + slice_xrn is not None. + + :param slice_xrn: srn of the slice + :type slice_xrn: string + :param login: user'uid (ldap login) on cortexlab + :type login: string + :param version: can be set to sfa or cortexlab + :type version: RSpecVersion + :param options: used to specify if the leases should also be included in + the returned rspec. + :type options: dict + + :returns: Xml Rspec. + :rtype: XML + + + """ + + ldap_username = None + rspec = None + version_manager = VersionManager() + version = version_manager.get_version(version) + logger.debug("CortexlabAggregate \t get_rspec ***version %s \ + version.type %s version.version %s options %s \r\n" + % (version, version.type, version.version, options)) + + if slice_xrn is None: + rspec_version = version_manager._get_version(version.type, + version.version, 'ad') + + else: + rspec_version = version_manager._get_version( + version.type, version.version, 'manifest') + + slices, slivers = self.get_slice_and_slivers(slice_xrn, login) + if slice_xrn and slices is not None: + #Get user associated with this slice + #for one_slice in slices : + ldap_username = slices[0]['reg_researchers'][0].__dict__['hrn'] + # ldap_username = slices[0]['user'] + tmp = ldap_username.split('.') + ldap_username = tmp[1] + logger.debug("CortexlabAggregate \tget_rspec **** \ + LDAP USERNAME %s \r\n" \ + % (ldap_username)) + #at this point sliver may be empty if no cortexlab job + #is running for this user/slice. + rspec = RSpec(version=rspec_version, user_options=options) + + logger.debug("\r\n \r\n CortexlabAggregate \tget_rspec *** \ + slice_xrn %s slices %s\r\n \r\n" + % (slice_xrn, slices)) + + if options is not None and 'list_leases' in options: + lease_option = options['list_leases'] + else: + #If no options are specified, at least print the resources + lease_option = 'all' + #if slice_xrn : + #lease_option = 'all' + + if lease_option in ['all', 'resources']: + #if not options.get('list_leases') or options.get('list_leases') + #and options['list_leases'] != 'leases': + nodes = self.get_nodes(slices, slivers) + if slice_xrn and slices is None: + nodes = [] + logger.debug("\r\n") + logger.debug("CortexlabAggregate \t lease_option %s \ + get rspec ******* nodes %s" + % (lease_option, nodes)) + + sites_set = set([node['location']['site'] for node in nodes]) + + #In case creating a job, slice_xrn is not set to None + rspec.version.add_nodes(nodes) + if slice_xrn and slices is not None: + # #Get user associated with this slice + # #for one_slice in slices : + # ldap_username = slices[0]['reg_researchers'] + # # ldap_username = slices[0]['user'] + # tmp = ldap_username.split('.') + # ldap_username = tmp[1] + # # ldap_username = tmp[1].split('_')[0] + + logger.debug("CortexlabAggregate \tget_rspec **** \ + version type %s ldap_ user %s \r\n" \ + % (version.type, ldap_username)) + if version.type == "Iotlab": + rspec.version.add_connection_information( + ldap_username, sites_set) + + default_sliver = slivers.get('default_sliver', []) + if default_sliver and len(nodes) is not 0: + #default_sliver_attribs = default_sliver.get('tags', []) + logger.debug("CortexlabAggregate \tget_rspec **** \ + default_sliver%s \r\n" % (default_sliver)) + for attrib in default_sliver: + rspec.version.add_default_sliver_attribute( + attrib, default_sliver[attrib]) + + if lease_option in ['all','leases']: + leases = self.get_all_leases(ldap_username) + rspec.version.add_leases(leases) + logger.debug("CortexlabAggregate \tget_rspec **** \ + FINAL RSPEC %s \r\n" % (rspec.toxml())) + return rspec.toxml() diff --git a/sfa/cortexlab/cortexlabapi.py b/sfa/cortexlab/cortexlabapi.py new file mode 100644 index 00000000..4635b1b3 --- /dev/null +++ b/sfa/cortexlab/cortexlabapi.py @@ -0,0 +1,1420 @@ +""" +File containing the IotlabTestbedAPI, used to interact with nodes, users, +slices, leases and keys, as well as the dedicated iotlab database and table, +holding information about which slice is running which job. + +""" +from datetime import datetime + +from sfa.util.sfalogging import logger + +from sfa.storage.alchemy import dbsession +from sqlalchemy.orm import joinedload +from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey + +from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB, LeaseTableXP +from sfa.cortexlab.LDAPapi import LDAPapi + +from sfa.util.xrn import Xrn, hrn_to_urn, get_authority + +from sfa.trust.certificate import Keypair, convert_public_key +from sfa.trust.gid import create_uuid +from sfa.trust.hierarchy import Hierarchy + +from sfa.iotlab.iotlabaggregate import iotlab_xrn_object + +class CortexlabTestbedAPI(): + """ Class enabled to use LDAP and OAR api calls. """ + + _MINIMUM_DURATION = 10 # 10 units of granularity 60 s, 10 mins + + def __init__(self, config): + """Creates an instance of OARrestapi and LDAPapi which will be used to + issue calls to OAR or LDAP methods. + Set the time format and the testbed granularity used for OAR + reservation and leases. + + :param config: configuration object from sfa.util.config + :type config: Config object + """ + self.cortexlab_leases_db = TestbedAdditionalSfaDB(config) + self.query_sites = CortexlabQueryNodes() + self.ldap = LDAPapi() + self.time_format = "%Y-%m-%d %H:%M:%S" + self.root_auth = config.SFA_REGISTRY_ROOT_AUTH + self.grain = 60 # 10 mins lease minimum, 60 sec granularity + #import logging, logging.handlers + #from sfa.util.sfalogging import _SfaLogger + #sql_logger = _SfaLogger(loggername = 'sqlalchemy.engine', \ + #level=logging.DEBUG) + return + + @staticmethod + def GetMinExperimentDurationInGranularity(): + """ Returns the minimum allowed duration for an experiment on the + testbed. In seconds. + + """ + return CortexlabTestbedAPI._MINIMUM_DURATION + + @staticmethod + def GetPeers(peer_filter=None ): + """ Gathers registered authorities in SFA DB and looks for specific peer + if peer_filter is specified. + :param peer_filter: name of the site authority looked for. + :type peer_filter: string + :returns: list of records. + + """ + + existing_records = {} + existing_hrns_by_types = {} + logger.debug("CORTEXLAB_API \tGetPeers peer_filter %s " % (peer_filter)) + all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all() + + for record in all_records: + existing_records[(record.hrn, record.type)] = record + if record.type not in existing_hrns_by_types: + existing_hrns_by_types[record.type] = [record.hrn] + else: + existing_hrns_by_types[record.type].append(record.hrn) + + logger.debug("CORTEXLAB_API \tGetPeer\texisting_hrns_by_types %s " + % (existing_hrns_by_types)) + records_list = [] + + try: + if peer_filter: + records_list.append(existing_records[(peer_filter, + 'authority')]) + else: + for hrn in existing_hrns_by_types['authority']: + records_list.append(existing_records[(hrn, 'authority')]) + + logger.debug("CORTEXLAB_API \tGetPeer \trecords_list %s " + % (records_list)) + + except KeyError: + pass + + return_records = records_list + logger.debug("CORTEXLAB_API \tGetPeer return_records %s " + % (return_records)) + return return_records + + #TODO : Handling OR request in make_ldap_filters_from_records + #instead of the for loop + #over the records' list + def GetPersons(self, person_filter=None): + """ + Get the enabled users and their properties from Cortexlab LDAP. + If a filter is specified, looks for the user whose properties match + the filter, otherwise returns the whole enabled users'list. + + :param person_filter: Must be a list of dictionnaries with users + properties when not set to None. + :type person_filter: list of dict + + :returns: Returns a list of users whose accounts are enabled + found in ldap. + :rtype: list of dicts + + """ + logger.debug("CORTEXLAB_API \tGetPersons person_filter %s" + % (person_filter)) + person_list = [] + if person_filter and isinstance(person_filter, list): + #If we are looking for a list of users (list of dict records) + #Usually the list contains only one user record + for searched_attributes in person_filter: + + #Get only enabled user accounts in iotlab LDAP : + #add a filter for make_ldap_filters_from_record + person = self.ldap.LdapFindUser(searched_attributes, + is_user_enabled=True) + #If a person was found, append it to the list + if person: + person_list.append(person) + + #If the list is empty, return None + if len(person_list) is 0: + person_list = None + + else: + #Get only enabled user accounts in iotlab LDAP : + #add a filter for make_ldap_filters_from_record + person_list = self.ldap.LdapFindUser(is_user_enabled=True) + + return person_list + + + + def DeleteOneLease(self, lease_id, username): + """ + + Deletes the lease with the specified lease_id and username on OAR by + posting a delete request to OAR. + + :param lease_id: Reservation identifier. + :param username: user's iotlab login in LDAP. + :type lease_id: Depends on what tou are using, could be integer or + string + :type username: string + + :returns: dictionary with the lease id and if delete has been successful + (True) or no (False) + :rtype: dict + + """ + + # Here delete the lease specified + answer = self.query_sites.delete_experiment(lease_id, username) + + # If the username is not necessary to delete the lease, then you can + # remove it from the parameters, given that you propagate the changes + # Return delete status so that you know if the delete has been + # successuf or not + + + if answer['status'] is True: + ret = {lease_id: True} + else: + ret = {lease_id: False} + logger.debug("CORTEXLAB_API \DeleteOneLease lease_id %s \r\n answer %s \ + username %s" % (lease_id, answer, username)) + return ret + + + + def GetNodesCurrentlyInUse(self): + """Returns a list of all the nodes involved in a currently running + experiment (and only the one not available at the moment the call to + this method is issued) + :rtype: list of nodes hostnames. + """ + node_hostnames_list = [] + return node_hostnames_list + + + def GetReservedNodes(self, username=None): + """ Get list of leases. Get the leases for the username if specified, + otherwise get all the leases. Finds the nodes hostnames for each + OAR node identifier. + :param username: user's LDAP login + :type username: string + :returns: list of reservations dict + :rtype: dict list + """ + + #Get the nodes in use and the reserved nodes + mandatory_sfa_keys = ['reserved_nodes','lease_id'] + reservation_dict_list = \ + self.query_sites.get_reserved_nodes(username = username) + + if len(reservation_dict_list) == 0: + return [] + + else: + # Ensure mandatory keys are in the dict + if not self.ensure_format_is_valid(reservation_dict_list, + mandatory_sfa_keys): + raise KeyError, "GetReservedNodes : Missing SFA mandatory keys" + + + return reservation_dict_list + + @staticmethod + def ensure_format_is_valid(list_dictionary_to_check, mandatory_keys_list): + for entry in list_dictionary_to_check: + if not all (key in entry for key in mandatory_keys_list): + return False + return True + + def GetNodes(self, node_filter_dict=None, return_fields_list=None): + """ + + Make a list of cortexlab nodes and their properties from information + given by ?. Search for specific nodes if some filters are + specified. Nodes properties returned if no return_fields_list given: + 'hrn','archi','mobile','hostname','site','boot_state','node_id', + 'radio','posx','posy,'posz'. + + :param node_filter_dict: dictionnary of lists with node properties. For + instance, if you want to look for a specific node with its hrn, + the node_filter_dict should be {'hrn': [hrn_of_the_node]} + :type node_filter_dict: dict + :param return_fields_list: list of specific fields the user wants to be + returned. + :type return_fields_list: list + :returns: list of dictionaries with node properties. Mandatory + properties hrn, site, hostname. Complete list (iotlab) ['hrn', + 'archi', 'mobile', 'hostname', 'site', 'mobility_type', + 'boot_state', 'node_id','radio', 'posx', 'posy', 'oar_id', 'posz'] + Radio, archi, mobile and position are useful to help users choose + the appropriate nodes. + :rtype: list + + :TODO: FILL IN THE BLANKS + """ + + # Here get full dict of nodes with all their properties. + mandatory_sfa_keys = ['hrn', 'site', 'hostname'] + node_list_dict = self.query_sites.get_all_nodes(node_filter_dict, + return_fields_list) + + if len(node_list_dict) == 0: + return_node_list = [] + + else: + # Ensure mandatory keys are in the dict + if not self.ensure_format_is_valid(node_list_dict, + mandatory_sfa_keys): + raise KeyError, "GetNodes : Missing SFA mandatory keys" + + + return_node_list = node_list_dict + return return_node_list + + + + @staticmethod + def AddSlice(slice_record, user_record): + """ + + Add slice to the local cortexlab sfa tables if the slice comes + from a federated site and is not yet in the cortexlab sfa DB, + although the user has already a LDAP login. + Called by verify_slice during lease/sliver creation. + + :param slice_record: record of slice, must contain hrn, gid, slice_id + and authority of the slice. + :type slice_record: dictionary + :param user_record: record of the user + :type user_record: RegUser + + """ + + sfa_record = RegSlice(hrn=slice_record['hrn'], + gid=slice_record['gid'], + pointer=slice_record['slice_id'], + authority=slice_record['authority']) + logger.debug("CORTEXLAB_API.PY AddSlice sfa_record %s user_record %s" + % (sfa_record, user_record)) + sfa_record.just_created() + dbsession.add(sfa_record) + dbsession.commit() + #Update the reg-researcher dependance table + sfa_record.reg_researchers = [user_record] + dbsession.commit() + + return + + + def GetSites(self, site_filter_name_list=None, return_fields_list=None): + """Returns the list of Cortexlab's sites with the associated nodes and + the sites' properties as dictionaries. Used in import. + + Site properties: + ['address_ids', 'slice_ids', 'name', 'node_ids', 'url', 'person_ids', + 'site_tag_ids', 'enabled', 'site', 'longitude', 'pcu_ids', + 'max_slivers', 'max_slices', 'ext_consortium_id', 'date_created', + 'latitude', 'is_public', 'peer_site_id', 'peer_id', 'abbreviated_name'] + can be empty ( []): address_ids, slice_ids, pcu_ids, person_ids, + site_tag_ids + + :param site_filter_name_list: used to specify specific sites + :param return_fields_list: field that has to be returned + :type site_filter_name_list: list + :type return_fields_list: list + :rtype: list of dicts + + """ + site_list_dict = self.query_sites.get_sites(site_filter_name_list, + return_fields_list) + + mandatory_sfa_keys = ['name', 'node_ids', 'longitude','site' ] + + if len(site_list_dict) == 0: + return_site_list = [] + + else: + # Ensure mandatory keys are in the dict + if not self.ensure_format_is_valid(site_list_dict, + mandatory_sfa_keys): + raise KeyError, "GetSites : Missing sfa mandatory keys" + + return_site_list = site_list_dict + return return_site_list + + + #TODO : Check rights to delete person + def DeletePerson(self, person_record): + """Disable an existing account in cortexlab LDAP. + + Users and techs can only delete themselves. PIs can only + delete themselves and other non-PIs at their sites. + ins can delete anyone. + + :param person_record: user's record + :type person_record: dict + :returns: True if successful, False otherwise. + :rtype: boolean + + .. todo:: CHECK THAT ONLY THE USER OR ADMIN CAN DEL HIMSELF. + """ + #Disable user account in iotlab LDAP + ret = self.ldap.LdapMarkUserAsDeleted(person_record) + logger.warning("CORTEXLAB_API DeletePerson %s " % (person_record)) + return ret['bool'] + + def DeleteSlice(self, slice_record): + """Deletes the specified slice and kills the jobs associated with + the slice if any, using DeleteSliceFromNodes. + + :param slice_record: record of the slice, must contain experiment_id, user + :type slice_record: dict + :returns: True if all the jobs in the slice have been deleted, + or the list of jobs that could not be deleted otherwise. + :rtype: list or boolean + + .. seealso:: DeleteSliceFromNodes + + """ + ret = self.DeleteSliceFromNodes(slice_record) + delete_failed = None + for experiment_id in ret: + if False in ret[experiment_id]: + if delete_failed is None: + delete_failed = [] + delete_failed.append(experiment_id) + + logger.info("CORTEXLAB_API DeleteSlice %s answer %s"%(slice_record, \ + delete_failed)) + return delete_failed or True + + @staticmethod + def __add_person_to_db(user_dict): + """ + Add a federated user straight to db when the user issues a lease + request with iotlab nodes and that he has not registered with cortexlab + yet (that is he does not have a LDAP entry yet). + Uses parts of the routines in CortexlabImport when importing user + from LDAP. + Called by AddPerson, right after LdapAddUser. + :param user_dict: Must contain email, hrn and pkey to get a GID + and be added to the SFA db. + :type user_dict: dict + + """ + check_if_exists = \ + dbsession.query(RegUser).filter_by(email = user_dict['email']).first() + #user doesn't exists + if not check_if_exists: + logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \ + " %(user_dict)) + hrn = user_dict['hrn'] + person_urn = hrn_to_urn(hrn, 'user') + pubkey = user_dict['pkey'] + try: + pkey = convert_public_key(pubkey) + except TypeError: + #key not good. create another pkey + logger.warn('__add_person_to_db: unable to convert public \ + key for %s' %(hrn )) + pkey = Keypair(create=True) + + + if pubkey is not None and pkey is not None : + hierarchy = Hierarchy() + person_gid = hierarchy.create_gid(person_urn, create_uuid(), \ + pkey) + if user_dict['email']: + logger.debug("__add_person_to_db \r\n \r\n \ + IOTLAB IMPORTER PERSON EMAIL OK email %s "\ + %(user_dict['email'])) + person_gid.set_email(user_dict['email']) + + user_record = RegUser(hrn=hrn , pointer= '-1', \ + authority=get_authority(hrn), \ + email=user_dict['email'], gid = person_gid) + user_record.reg_keys = [RegKey(user_dict['pkey'])] + user_record.just_created() + dbsession.add (user_record) + dbsession.commit() + return + + + def AddPerson(self, record): + """ + + Adds a new account. Any fields specified in records are used, + otherwise defaults are used. Creates an appropriate login by calling + LdapAddUser. + + :param record: dictionary with the sfa user's properties. + :returns: a dicitonary with the status. If successful, the dictionary + boolean is set to True and there is a 'uid' key with the new login + added to LDAP, otherwise the bool is set to False and a key + 'message' is in the dictionary, with the error message. + :rtype: dict + + """ + ret = self.ldap.LdapAddUser(record) + + if ret['bool'] is True: + record['hrn'] = self.root_auth + '.' + ret['uid'] + logger.debug("CORTEXLAB_API AddPerson return code %s record %s " + % (ret, record)) + self.__add_person_to_db(record) + return ret + + + + + + #TODO AddPersonKey 04/07/2012 SA + def AddPersonKey(self, person_uid, old_attributes_dict, new_key_dict): + """Adds a new key to the specified account. Adds the key to the + iotlab ldap, provided that the person_uid is valid. + + Non-admins can only modify their own keys. + + :param person_uid: user's iotlab login in LDAP + :param old_attributes_dict: dict with the user's old sshPublicKey + :param new_key_dict: dict with the user's new sshPublicKey + :type person_uid: string + + + :rtype: Boolean + :returns: True if the key has been modified, False otherwise. + + """ + ret = self.ldap.LdapModify(person_uid, old_attributes_dict, \ + new_key_dict) + logger.warning("CORTEXLAB_API AddPersonKey EMPTY - DO NOTHING \r\n ") + return ret['bool'] + + + @staticmethod + def _process_walltime(duration): + """ Calculates the walltime in seconds from the duration in H:M:S + specified in the RSpec. + + """ + if duration: + # Fixing the walltime by adding a few delays. + # First put the walltime in seconds oarAdditionalDelay = 20; + # additional delay for /bin/sleep command to + # take in account prologue and epilogue scripts execution + # int walltimeAdditionalDelay = 240; additional delay + #for prologue/epilogue execution = $SERVER_PROLOGUE_EPILOGUE_TIMEOUT + #in oar.conf + # Put the duration in seconds first + #desired_walltime = duration * 60 + desired_walltime = duration + total_walltime = desired_walltime + 240 #+4 min Update SA 23/10/12 + sleep_walltime = desired_walltime # 0 sec added Update SA 23/10/12 + walltime = [] + #Put the walltime back in str form + #First get the hours + walltime.append(str(total_walltime / 3600)) + total_walltime = total_walltime - 3600 * int(walltime[0]) + #Get the remaining minutes + walltime.append(str(total_walltime / 60)) + total_walltime = total_walltime - 60 * int(walltime[1]) + #Get the seconds + walltime.append(str(total_walltime)) + + else: + logger.log_exc(" __process_walltime duration null") + + return walltime, sleep_walltime + + @staticmethod + def _create_job_structure_request_for_OAR(lease_dict): + """ Creates the structure needed for a correct POST on OAR. + Makes the timestamp transformation into the appropriate format. + Sends the POST request to create the job with the resources in + added_nodes. + + """ + + nodeid_list = [] + reqdict = {} + + + reqdict['workdir'] = '/tmp' + reqdict['resource'] = "{network_address in (" + + for node in lease_dict['added_nodes']: + logger.debug("\r\n \r\n OARrestapi \t \ + __create_job_structure_request_for_OAR node %s" %(node)) + + # Get the ID of the node + nodeid = node + reqdict['resource'] += "'" + nodeid + "', " + nodeid_list.append(nodeid) + + custom_length = len(reqdict['resource'])- 2 + reqdict['resource'] = reqdict['resource'][0:custom_length] + \ + ")}/nodes=" + str(len(nodeid_list)) + + + walltime, sleep_walltime = \ + IotlabTestbedAPI._process_walltime(\ + int(lease_dict['lease_duration'])) + + + reqdict['resource'] += ",walltime=" + str(walltime[0]) + \ + ":" + str(walltime[1]) + ":" + str(walltime[2]) + reqdict['script_path'] = "/bin/sleep " + str(sleep_walltime) + + #In case of a scheduled experiment (not immediate) + #To run an XP immediately, don't specify date and time in RSpec + #They will be set to None. + if lease_dict['lease_start_time'] is not '0': + #Readable time accepted by OAR + start_time = datetime.fromtimestamp( \ + int(lease_dict['lease_start_time'])).\ + strftime(lease_dict['time_format']) + reqdict['reservation'] = start_time + #If there is not start time, Immediate XP. No need to add special + # OAR parameters + + + reqdict['type'] = "deploy" + reqdict['directory'] = "" + reqdict['name'] = "SFA_" + lease_dict['slice_user'] + + return reqdict + + + def LaunchExperimentOnTestbed(self, added_nodes, slice_name, \ + lease_start_time, lease_duration, slice_user=None): + + """ + Create an experiment request structure based on the information provided + and schedule/run the experiment on the testbed by reserving the nodes. + :param added_nodes: list of nodes that belong to the described lease. + :param slice_name: the slice hrn associated to the lease. + :param lease_start_time: timestamp of the lease startting time. + :param lease_duration: lease duration in minutes + + """ + lease_dict = {} + # Add in the dict whatever is necessary to create the experiment on + # the testbed + lease_dict['lease_start_time'] = lease_start_time + lease_dict['lease_duration'] = lease_duration + lease_dict['added_nodes'] = added_nodes + lease_dict['slice_name'] = slice_name + lease_dict['slice_user'] = slice_user + lease_dict['grain'] = self.GetLeaseGranularity() + + + + answer = self.query_sites.schedule_experiment(lease_dict) + try: + experiment_id = answer['id'] + except KeyError: + logger.log_exc("CORTEXLAB_API \tLaunchExperimentOnTestbed \ + Impossible to create xp %s " %(answer)) + return None + + if experiment_id : + logger.debug("CORTEXLAB_API \tLaunchExperimentOnTestbed \ + experiment_id %s added_nodes %s slice_user %s" + %(experiment_id, added_nodes, slice_user)) + + + return experiment_id + + + def AddLeases(self, hostname_list, slice_record, + lease_start_time, lease_duration): + + """Creates an experiment on the testbed corresponding to the information + provided as parameters. Adds the experiment id and the slice hrn in the + lease table on the additional sfa database so that we are able to know + which slice has which nodes. + + :param hostname_list: list of nodes' OAR hostnames. + :param slice_record: sfa slice record, must contain login and hrn. + :param lease_start_time: starting time , unix timestamp format + :param lease_duration: duration in minutes + + :type hostname_list: list + :type slice_record: dict + :type lease_start_time: integer + :type lease_duration: integer + + """ + logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases hostname_list %s \ + slice_record %s lease_start_time %s lease_duration %s "\ + %( hostname_list, slice_record , lease_start_time, \ + lease_duration)) + + username = slice_record['login'] + + experiment_id = self.LaunchExperimentOnTestbed(hostname_list, \ + slice_record['hrn'], \ + lease_start_time, lease_duration, \ + username) + start_time = \ + datetime.fromtimestamp(int(lease_start_time)).\ + strftime(self.time_format) + end_time = lease_start_time + lease_duration + + + logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases TURN ON LOGGING SQL \ + %s %s %s "%(slice_record['hrn'], experiment_id, end_time)) + + + logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases %s %s %s " \ + %(type(slice_record['hrn']), type(experiment_id), + type(end_time))) + + testbed_xp_row = LeaseTableXP(slice_hrn=slice_record['hrn'], + experiment_id=experiment_id, end_time=end_time) + + logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases testbed_xp_row %s" \ + %(testbed_xp_row)) + self.cortexlab_leases_db.testbed_session.add(testbed_xp_row) + self.cortexlab_leases_db.testbed_session.commit() + + logger.debug("CORTEXLAB_API \t AddLeases hostname_list start_time %s " \ + %(start_time)) + + return + + def DeleteLeases(self, leases_id_list, slice_hrn): + """ + + Deletes several leases, based on their experiment ids and the slice + they are associated with. Uses DeleteOneLease to delete the + experiment on the testbed. Note that one slice can contain multiple + experiments, and in this + case all the experiments in the leases_id_list MUST belong to this + same slice, since there is only one slice hrn provided here. + + :param leases_id_list: list of job ids that belong to the slice whose + slice hrn is provided. + :param slice_hrn: the slice hrn. + :type slice_hrn: string + + .. warning:: Does not have a return value since there was no easy + way to handle failure when dealing with multiple job delete. Plus, + there was no easy way to report it to the user. + + """ + logger.debug("CORTEXLAB_API DeleteLeases leases_id_list %s slice_hrn %s \ + \r\n " %(leases_id_list, slice_hrn)) + for experiment_id in leases_id_list: + self.DeleteOneLease(experiment_id, slice_hrn) + + return + + #Delete the jobs from job_iotlab table + def DeleteSliceFromNodes(self, slice_record): + """ + Deletes all the running or scheduled jobs of a given slice + given its record. + + :param slice_record: record of the slice, must contain experiment_id, + user + :type slice_record: dict + :returns: dict of the jobs'deletion status. Success= True, Failure= + False, for each job id. + :rtype: dict + + .. note: used in driver delete_sliver + + """ + logger.debug("CORTEXLAB_API \t DeleteSliceFromNodes %s " + % (slice_record)) + + if isinstance(slice_record['experiment_id'], list): + experiment_bool_answer = {} + for experiment_id in slice_record['experiment_id']: + ret = self.DeleteOneLease(experiment_id, slice_record['user']) + + experiment_bool_answer.update(ret) + + else: + experiment_bool_answer = [self.DeleteOneLease( + slice_record['experiment_id'], + slice_record['user'])] + + return experiment_bool_answer + + + + def GetLeaseGranularity(self): + """ Returns the granularity of an experiment in the Iotlab testbed. + OAR uses seconds for experiments duration , the granulaity is also + defined in seconds. + Experiments which last less than 10 min (600 sec) are invalid""" + return self.grain + + + # @staticmethod + # def update_experiments_in_additional_sfa_db( job_oar_list, jobs_psql): + # """ Cleans the iotlab db by deleting expired and cancelled jobs. + # Compares the list of job ids given by OAR with the job ids that + # are already in the database, deletes the jobs that are no longer in + # the OAR job id list. + # :param job_oar_list: list of job ids coming from OAR + # :type job_oar_list: list + # :param job_psql: list of job ids cfrom the database. + # type job_psql: list + # """ + # #Turn the list into a set + # set_jobs_psql = set(jobs_psql) + + # kept_jobs = set(job_oar_list).intersection(set_jobs_psql) + # logger.debug ( "\r\n \t\ update_experiments_in_additional_sfa_db jobs_psql %s \r\n \t \ + # job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs)) + # deleted_jobs = set_jobs_psql.difference(kept_jobs) + # deleted_jobs = list(deleted_jobs) + # if len(deleted_jobs) > 0: + # self.cortexlab_leases_db.testbed_session.query(LeaseTableXP).filter(LeaseTableXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') + # self.cortexlab_leases_db.testbed_session.commit() + + # return + + @staticmethod + def filter_lease_name(reservation_list, filter_value): + filtered_reservation_list = list(reservation_list) + logger.debug("CORTEXLAB_API \t filter_lease_name reservation_list %s" \ + % (reservation_list)) + for reservation in reservation_list: + if 'slice_hrn' in reservation and \ + reservation['slice_hrn'] != filter_value: + filtered_reservation_list.remove(reservation) + + logger.debug("CORTEXLAB_API \t filter_lease_name filtered_reservation_list %s" \ + % (filtered_reservation_list)) + return filtered_reservation_list + + @staticmethod + def filter_lease_start_time(reservation_list, filter_value): + filtered_reservation_list = list(reservation_list) + + for reservation in reservation_list: + if 't_from' in reservation and \ + reservation['t_from'] > filter_value: + filtered_reservation_list.remove(reservation) + + return filtered_reservation_list + + def complete_leases_info(self, unfiltered_reservation_list, db_xp_dict): + + """Check that the leases list of dictionaries contains the appropriate + fields and piece of information here + :param unfiltered_reservation_list: list of leases to be completed. + :param db_xp_dict: leases information in the lease_sfa table + :returns local_unfiltered_reservation_list: list of leases completed. + list of dictionaries describing the leases, with all the needed + information (sfa,ldap,nodes)to identify one particular lease. + :returns testbed_xp_list: list of experiments'ids running or scheduled + on the testbed. + :rtype local_unfiltered_reservation_list: list of dict + :rtype testbed_xp_list: list + + """ + testbed_xp_list = [] + local_unfiltered_reservation_list = list(unfiltered_reservation_list) + # slice_hrn and lease_id are in the lease_table, + # so they are in the db_xp_dict. + # component_id_list : list of nodes xrns + # reserved_nodes : list of nodes' hostnames + # slice_id : slice urn, can be made from the slice hrn using hrn_to_urn + for resa in local_unfiltered_reservation_list: + + #Construct list of scheduled experiments (runing, waiting..) + testbed_xp_list.append(resa['lease_id']) + #If there is information on the experiment in the lease table + #(slice used and experiment id), meaning the experiment was created + # using sfa + if resa['lease_id'] in db_xp_dict: + xp_info = db_xp_dict[resa['lease_id']] + logger.debug("CORTEXLAB_API \tGetLeases xp_info %s" + % (xp_info)) + resa['slice_hrn'] = xp_info['slice_hrn'] + resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice') + + #otherwise, assume it is a cortexlab slice, created via the + # cortexlab portal + else: + resa['slice_id'] = hrn_to_urn(self.root_auth + '.' + + resa['user'] + "_slice", 'slice') + resa['slice_hrn'] = Xrn(resa['slice_id']).get_hrn() + + resa['component_id_list'] = [] + #Transform the hostnames into urns (component ids) + for node in resa['reserved_nodes']: + + iotlab_xrn = iotlab_xrn_object(self.root_auth, node) + resa['component_id_list'].append(iotlab_xrn.urn) + + return local_unfiltered_reservation_list, testbed_xp_list + + def GetLeases(self, lease_filter_dict=None, login=None): + """ + + Get the list of leases from the testbed with complete information + about in which slice is running which experiment ans which nodes are + involved. + Two purposes: + -Fetch all the experiments from the testbed (running, waiting..) + complete the reservation information with slice hrn + found in testbed_xp table. If not available in the table, + assume it is a cortexlab slice. + -Updates the cortexlab table, deleting jobs when necessary. + + :returns: reservation_list, list of dictionaries with 'lease_id', + 'reserved_nodes','slice_id','user', 'component_id_list', + 'slice_hrn', 'resource_ids', 't_from', 't_until'. Other + keys can be returned if necessary, such as the 'state' of the lease, + if the information has been added in GetReservedNodes. + :rtype: list + + """ + + unfiltered_reservation_list = self.GetReservedNodes(login) + + reservation_list = [] + #Find the slice associated with this user ldap uid + logger.debug(" CORTEXLAB_API.PY \tGetLeases login %s\ + unfiltered_reservation_list %s " + % (login, unfiltered_reservation_list)) + #Create user dict first to avoid looking several times for + #the same user in LDAP SA 27/07/12 + + + db_xp_query = self.cortexlab_leases_db.testbed_session.query(LeaseTableXP).all() + db_xp_dict = dict([(row.experiment_id, row.__dict__) + for row in db_xp_query]) + + logger.debug("CORTEXLAB_API \tGetLeases db_xp_dict %s" + % (db_xp_dict)) + db_xp_id_list = [row.experiment_id for row in db_xp_query] + + required_fiels_in_leases = ['lease_id', + 'reserved_nodes','slice_id', 'user', 'component_id_list', + 'slice_hrn', 'resource_ids', 't_from', 't_until'] + + # Add any missing information on the leases with complete_leases_info + unfiltered_reservation_list, testbed_xp_list = \ + self.complete_leases_info(unfiltered_reservation_list, + db_xp_dict) + # Check that the list of leases is complete and have the mandatory + # information + format_status = self.ensure_format_is_valid(unfiltered_reservation_list, + required_fiels_in_leases) + + if not format_status: + logger.log_exc("\tCortexlabapi \t GetLeases : Missing fields in \ + reservation list") + raise KeyError, "GetLeases : Missing fields in reservation list " + + if lease_filter_dict: + logger.debug("CORTEXLAB_API \tGetLeases \ + \r\n leasefilter %s" % ( lease_filter_dict)) + + filter_dict_functions = { + 'slice_hrn' : CortexlabTestbedAPI.filter_lease_name, + 't_from' : CortexlabTestbedAPI.filter_lease_start_time + } + + reservation_list = list(unfiltered_reservation_list) + for filter_type in lease_filter_dict: + logger.debug("CORTEXLAB_API \tGetLeases reservation_list %s" \ + % (reservation_list)) + reservation_list = filter_dict_functions[filter_type](\ + reservation_list,lease_filter_dict[filter_type] ) + + + if lease_filter_dict is None: + reservation_list = unfiltered_reservation_list + + self.cortexlab_leases_db.update_experiments_in_additional_sfa_db( + testbed_xp_list, db_xp_id_list) + + logger.debug(" CORTEXLAB_API.PY \tGetLeases reservation_list %s" + % (reservation_list)) + return reservation_list + + + + +#TODO FUNCTIONS SECTION 04/07/2012 SA + + ##TODO : Is UnBindObjectFromPeer still necessary ? Currently does nothing + ##04/07/2012 SA + #@staticmethod + #def UnBindObjectFromPeer( auth, object_type, object_id, shortname): + #""" This method is a hopefully temporary hack to let the sfa correctly + #detach the objects it creates from a remote peer object. This is + #needed so that the sfa federation link can work in parallel with + #RefreshPeer, as RefreshPeer depends on remote objects being correctly + #marked. + #Parameters: + #auth : struct, API authentication structure + #AuthMethod : string, Authentication method to use + #object_type : string, Object type, among 'site','person','slice', + #'node','key' + #object_id : int, object_id + #shortname : string, peer shortname + #FROM PLC DOC + + #""" + #logger.warning("CORTEXLAB_API \tUnBindObjectFromPeer EMPTY-\ + #DO NOTHING \r\n ") + #return + + ##TODO Is BindObjectToPeer still necessary ? Currently does nothing + ##04/07/2012 SA + #|| Commented out 28/05/13 SA + #def BindObjectToPeer(self, auth, object_type, object_id, shortname=None, \ + #remote_object_id=None): + #"""This method is a hopefully temporary hack to let the sfa correctly + #attach the objects it creates to a remote peer object. This is needed + #so that the sfa federation link can work in parallel with RefreshPeer, + #as RefreshPeer depends on remote objects being correctly marked. + #Parameters: + #shortname : string, peer shortname + #remote_object_id : int, remote object_id, set to 0 if unknown + #FROM PLC API DOC + + #""" + #logger.warning("CORTEXLAB_API \tBindObjectToPeer EMPTY - DO NOTHING \r\n ") + #return + + ##TODO UpdateSlice 04/07/2012 SA || Commented out 28/05/13 SA + ##Funciton should delete and create another job since oin iotlab slice=job + #def UpdateSlice(self, auth, slice_id_or_name, slice_fields=None): + #"""Updates the parameters of an existing slice with the values in + #slice_fields. + #Users may only update slices of which they are members. + #PIs may update any of the slices at their sites, or any slices of + #which they are members. Admins may update any slice. + #Only PIs and admins may update max_nodes. Slices cannot be renewed + #(by updating the expires parameter) more than 8 weeks into the future. + #Returns 1 if successful, faults otherwise. + #FROM PLC API DOC + + #""" + #logger.warning("CORTEXLAB_API UpdateSlice EMPTY - DO NOTHING \r\n ") + #return + + #Unused SA 30/05/13, we only update the user's key or we delete it. + ##TODO UpdatePerson 04/07/2012 SA + #def UpdatePerson(self, iotlab_hrn, federated_hrn, person_fields=None): + #"""Updates a person. Only the fields specified in person_fields + #are updated, all other fields are left untouched. + #Users and techs can only update themselves. PIs can only update + #themselves and other non-PIs at their sites. + #Returns 1 if successful, faults otherwise. + #FROM PLC API DOC + + #""" + ##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn) + ##self.cortexlab_leases_db.testbed_session.add(new_row) + ##self.cortexlab_leases_db.testbed_session.commit() + + #logger.debug("CORTEXLAB_API UpdatePerson EMPTY - DO NOTHING \r\n ") + #return + + @staticmethod + def GetKeys(key_filter=None): + """Returns a dict of dict based on the key string. Each dict entry + contains the key id, the ssh key, the user's email and the + user's hrn. + If key_filter is specified and is an array of key identifiers, + only keys matching the filter will be returned. + + Admin may query all keys. Non-admins may only query their own keys. + FROM PLC API DOC + + :returns: dict with ssh key as key and dicts as value. + :rtype: dict + """ + if key_filter is None: + keys = dbsession.query(RegKey).options(joinedload('reg_user')).all() + else: + keys = dbsession.query(RegKey).options(joinedload('reg_user')).filter(RegKey.key.in_(key_filter)).all() + + key_dict = {} + for key in keys: + key_dict[key.key] = {'key_id': key.key_id, 'key': key.key, + 'email': key.reg_user.email, + 'hrn': key.reg_user.hrn} + + #ldap_rslt = self.ldap.LdapSearch({'enabled']=True}) + #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \ + #for user in ldap_rslt) + + logger.debug("CORTEXLAB_API GetKeys -key_dict %s \r\n " % (key_dict)) + return key_dict + + #TODO : test + def DeleteKey(self, user_record, key_string): + """Deletes a key in the LDAP entry of the specified user. + + Removes the key_string from the user's key list and updates the LDAP + user's entry with the new key attributes. + + :param key_string: The ssh key to remove + :param user_record: User's record + :type key_string: string + :type user_record: dict + :returns: True if sucessful, False if not. + :rtype: Boolean + + """ + all_user_keys = user_record['keys'] + all_user_keys.remove(key_string) + new_attributes = {'sshPublicKey':all_user_keys} + ret = self.ldap.LdapModifyUser(user_record, new_attributes) + logger.debug("CORTEXLAB_API DeleteKey %s- " % (ret)) + return ret['bool'] + + + + + @staticmethod + def _sql_get_slice_info(slice_filter): + """ + Get the slice record based on the slice hrn. Fetch the record of the + user associated with the slice by using joinedload based on the + reg_researcher relationship. + + :param slice_filter: the slice hrn we are looking for + :type slice_filter: string + :returns: the slice record enhanced with the user's information if the + slice was found, None it wasn't. + + :rtype: dict or None. + """ + #DO NOT USE RegSlice - reg_researchers to get the hrn + #of the user otherwise will mess up the RegRecord in + #Resolve, don't know why - SA 08/08/2012 + + #Only one entry for one user = one slice in testbed_xp table + #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() + raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first() + #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() + if raw_slicerec: + #load_reg_researcher + #raw_slicerec.reg_researchers + raw_slicerec = raw_slicerec.__dict__ + logger.debug(" CORTEXLAB_API \t _sql_get_slice_info slice_filter %s \ + raw_slicerec %s" % (slice_filter, raw_slicerec)) + slicerec = raw_slicerec + #only one researcher per slice so take the first one + #slicerec['reg_researchers'] = raw_slicerec['reg_researchers'] + #del slicerec['reg_researchers']['_sa_instance_state'] + return slicerec + + else: + return None + + @staticmethod + def _sql_get_slice_info_from_user(slice_filter): + """ + Get the slice record based on the user recordid by using a joinedload + on the relationship reg_slices_as_researcher. Format the sql record + into a dict with the mandatory fields for user and slice. + :returns: dict with slice record and user record if the record was found + based on the user's id, None if not.. + :rtype:dict or None.. + """ + #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() + raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id=slice_filter).first() + #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() + #Put it in correct order + user_needed_fields = ['peer_authority', 'hrn', 'last_updated', + 'classtype', 'authority', 'gid', 'record_id', + 'date_created', 'type', 'email', 'pointer'] + slice_needed_fields = ['peer_authority', 'hrn', 'last_updated', + 'classtype', 'authority', 'gid', 'record_id', + 'date_created', 'type', 'pointer'] + if raw_slicerec: + #raw_slicerec.reg_slices_as_researcher + raw_slicerec = raw_slicerec.__dict__ + slicerec = {} + slicerec = \ + dict([(k, raw_slicerec[ + 'reg_slices_as_researcher'][0].__dict__[k]) + for k in slice_needed_fields]) + slicerec['reg_researchers'] = dict([(k, raw_slicerec[k]) + for k in user_needed_fields]) + #TODO Handle multiple slices for one user SA 10/12/12 + #for now only take the first slice record associated to the rec user + ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__ + #del raw_slicerec['reg_slices_as_researcher'] + #slicerec['reg_researchers'] = raw_slicerec + ##del slicerec['_sa_instance_state'] + + return slicerec + + else: + return None + + def _get_slice_records(self, slice_filter=None, + slice_filter_type=None): + """ + Get the slice record depending on the slice filter and its type. + :param slice_filter: Can be either the slice hrn or the user's record + id. + :type slice_filter: string + :param slice_filter_type: describes the slice filter type used, can be + slice_hrn or record_id_user + :type: string + :returns: the slice record + :rtype:dict + .. seealso::_sql_get_slice_info_from_user + .. seealso:: _sql_get_slice_info + """ + + #Get list of slices based on the slice hrn + if slice_filter_type == 'slice_hrn': + + #if get_authority(slice_filter) == self.root_auth: + #login = slice_filter.split(".")[1].split("_")[0] + + slicerec = self._sql_get_slice_info(slice_filter) + + if slicerec is None: + return None + #return login, None + + #Get slice based on user id + if slice_filter_type == 'record_id_user': + + slicerec = self._sql_get_slice_info_from_user(slice_filter) + + if slicerec: + fixed_slicerec_dict = slicerec + #At this point if there is no login it means + #record_id_user filter has been used for filtering + #if login is None : + ##If theslice record is from iotlab + #if fixed_slicerec_dict['peer_authority'] is None: + #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0] + #return login, fixed_slicerec_dict + return fixed_slicerec_dict + else: + return None + + + def GetSlices(self, slice_filter=None, slice_filter_type=None, + login=None): + """Get the slice records from the sfa db and add lease information + if any. + + :param slice_filter: can be the slice hrn or slice record id in the db + depending on the slice_filter_type. + :param slice_filter_type: defines the type of the filtering used, Can be + either 'slice_hrn' or 'record_id'. + :type slice_filter: string + :type slice_filter_type: string + :returns: a slice dict if slice_filter and slice_filter_type + are specified and a matching entry is found in the db. The result + is put into a list.Or a list of slice dictionnaries if no filters + arespecified. + + :rtype: list + + """ + #login = None + authorized_filter_types_list = ['slice_hrn', 'record_id_user'] + return_slicerec_dictlist = [] + + #First try to get information on the slice based on the filter provided + if slice_filter_type in authorized_filter_types_list: + fixed_slicerec_dict = self._get_slice_records(slice_filter, + slice_filter_type) + # if the slice was not found in the sfa db + if fixed_slicerec_dict is None: + return return_slicerec_dictlist + + slice_hrn = fixed_slicerec_dict['hrn'] + + logger.debug(" CORTEXLAB_API \tGetSlices login %s \ + slice record %s slice_filter %s \ + slice_filter_type %s " % (login, + fixed_slicerec_dict, slice_filter, + slice_filter_type)) + + + #Now we have the slice record fixed_slicerec_dict, get the + #jobs associated to this slice + leases_list = [] + + leases_list = self.GetLeases(login=login) + #If no job is running or no job scheduled + #return only the slice record + if leases_list == [] and fixed_slicerec_dict: + return_slicerec_dictlist.append(fixed_slicerec_dict) + + # if the jobs running don't belong to the user/slice we are looking + # for + leases_hrn = [lease['slice_hrn'] for lease in leases_list] + if slice_hrn not in leases_hrn: + return_slicerec_dictlist.append(fixed_slicerec_dict) + #If several experiments for one slice , put the slice record into + # each lease information dict + for lease in leases_list: + slicerec_dict = {} + logger.debug("CORTEXLAB_API.PY \tGetSlices slice_filter %s \ + \t lease['slice_hrn'] %s" + % (slice_filter, lease['slice_hrn'])) + if lease['slice_hrn'] == slice_hrn: + slicerec_dict['experiment_id'] = lease['lease_id'] + #Update lease dict with the slice record + if fixed_slicerec_dict: + fixed_slicerec_dict['experiment_id'] = [] + fixed_slicerec_dict['experiment_id'].append( + slicerec_dict['experiment_id']) + slicerec_dict.update(fixed_slicerec_dict) + + slicerec_dict['slice_hrn'] = lease['slice_hrn'] + slicerec_dict['hrn'] = lease['slice_hrn'] + slicerec_dict['user'] = lease['user'] + slicerec_dict.update( + {'list_node_ids': + {'hostname': lease['reserved_nodes']}}) + slicerec_dict.update({'node_ids': lease['reserved_nodes']}) + + + return_slicerec_dictlist.append(slicerec_dict) + + + logger.debug("CORTEXLAB_API.PY \tGetSlices \ + slicerec_dict %s return_slicerec_dictlist %s \ + lease['reserved_nodes'] \ + %s" % (slicerec_dict, return_slicerec_dictlist, + lease['reserved_nodes'])) + + logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \ + return_slicerec_dictlist %s" + % (return_slicerec_dictlist)) + + return return_slicerec_dictlist + + + else: + #Get all slices from the cortexlab sfa database , get the user info + # as well at the same time put them in dict format + + query_slice_list = \ + dbsession.query(RegSlice).options(joinedload('reg_researchers')).all() + + for record in query_slice_list: + tmp = record.__dict__ + tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__ + return_slicerec_dictlist.append(tmp) + + + #Get all the experiments reserved nodes + leases_list = self.GetReservedNodes() + + for fixed_slicerec_dict in return_slicerec_dictlist: + slicerec_dict = {} + #Check if the slice belongs to a cortexlab user + if fixed_slicerec_dict['peer_authority'] is None: + owner = fixed_slicerec_dict['hrn'].split( + ".")[1].split("_")[0] + else: + owner = None + for lease in leases_list: + if owner == lease['user']: + slicerec_dict['experiment_id'] = lease['lease_id'] + + #for reserved_node in lease['reserved_nodes']: + logger.debug("CORTEXLAB_API.PY \tGetSlices lease %s " + % (lease)) + slicerec_dict.update(fixed_slicerec_dict) + slicerec_dict.update({'node_ids': + lease['reserved_nodes']}) + slicerec_dict.update({'list_node_ids': + {'hostname': + lease['reserved_nodes']}}) + + + fixed_slicerec_dict.update(slicerec_dict) + + logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \ + return_slicerec_dictlist %s \slice_filter %s " \ + %(return_slicerec_dictlist, slice_filter)) + + return return_slicerec_dictlist + + + + #Update slice unused, therefore sfa_fields_to_iotlab_fields unused + #SA 30/05/13 + #@staticmethod + #def sfa_fields_to_iotlab_fields(sfa_type, hrn, record): + #""" + #""" + + #iotlab_record = {} + ##for field in record: + ## iotlab_record[field] = record[field] + + #if sfa_type == "slice": + ##instantion used in get_slivers ? + #if not "instantiation" in iotlab_record: + #iotlab_record["instantiation"] = "iotlab-instantiated" + ##iotlab_record["hrn"] = hrn_to_pl_slicename(hrn) + ##Unused hrn_to_pl_slicename because Iotlab's hrn already + ##in the appropriate form SA 23/07/12 + #iotlab_record["hrn"] = hrn + #logger.debug("CORTEXLAB_API.PY sfa_fields_to_iotlab_fields \ + #iotlab_record %s " %(iotlab_record['hrn'])) + #if "url" in record: + #iotlab_record["url"] = record["url"] + #if "description" in record: + #iotlab_record["description"] = record["description"] + #if "expires" in record: + #iotlab_record["expires"] = int(record["expires"]) + + ##nodes added by OAR only and then imported to SFA + ##elif type == "node": + ##if not "hostname" in iotlab_record: + ##if not "hostname" in record: + ##raise MissingSfaInfo("hostname") + ##iotlab_record["hostname"] = record["hostname"] + ##if not "model" in iotlab_record: + ##iotlab_record["model"] = "geni" + + ##One authority only + ##elif type == "authority": + ##iotlab_record["login_base"] = hrn_to_iotlab_login_base(hrn) + + ##if not "name" in iotlab_record: + ##iotlab_record["name"] = hrn + + ##if not "abbreviated_name" in iotlab_record: + ##iotlab_record["abbreviated_name"] = hrn + + ##if not "enabled" in iotlab_record: + ##iotlab_record["enabled"] = True + + ##if not "is_public" in iotlab_record: + ##iotlab_record["is_public"] = True + + #return iotlab_record + + + + + + + + + + diff --git a/sfa/cortexlab/cortexlabdriver.py b/sfa/cortexlab/cortexlabdriver.py new file mode 100644 index 00000000..e99c725f --- /dev/null +++ b/sfa/cortexlab/cortexlabdriver.py @@ -0,0 +1,798 @@ +""" +Implements what a driver should provide for SFA to work. +""" +from sfa.util.faults import SliverDoesNotExist, UnknownSfaType +from sfa.util.sfalogging import logger +from sfa.storage.alchemy import dbsession +from sfa.storage.model import RegRecord + +from sfa.managers.driver import Driver +from sfa.rspecs.version_manager import VersionManager +from sfa.rspecs.rspec import RSpec + +from sfa.util.xrn import Xrn, hrn_to_urn, get_authority + +from sfa.iotlab.iotlabaggregate import IotlabAggregate, iotlab_xrn_to_hostname +from sfa.iotlab.iotlabslices import IotlabSlices + + +from sfa.iotlab.iotlabapi import IotlabTestbedAPI + + +class CortexlabDriver(Driver): + """ Cortexlab Driver class inherited from Driver generic class. + + Contains methods compliant with the SFA standard and the testbed + infrastructure (calls to LDAP and scheduler to book the nodes). + + .. seealso::: Driver class + + """ + def __init__(self, config): + """ + + Sets the iotlab SFA config parameters, + instanciates the testbed api and the iotlab database. + + :param config: iotlab SFA configuration object + :type config: Config object + + """ + Driver.__init__(self, config) + self.config = config + self.iotlab_api = IotlabTestbedAPI(config) + self.cache = None + + def augment_records_with_testbed_info(self, record_list): + """ + + Adds specific testbed info to the records. + + :param record_list: list of sfa dictionaries records + :type record_list: list + :returns: list of records with extended information in each record + :rtype: list + + """ + return self.fill_record_info(record_list) + + def fill_record_info(self, record_list): + """ + + For each SFA record, fill in the iotlab specific and SFA specific + fields in the record. + + :param record_list: list of sfa dictionaries records + :type record_list: list + :returns: list of records with extended information in each record + :rtype: list + + .. warning:: Should not be modifying record_list directly because modi + fication are kept outside the method's scope. Howerver, there is no + other way to do it given the way it's called in registry manager. + + """ + + logger.debug("IOTLABDRIVER \tfill_record_info records %s " + % (record_list)) + if not isinstance(record_list, list): + record_list = [record_list] + + try: + for record in record_list: + + if str(record['type']) == 'node': + # look for node info using GetNodes + # the record is about one node only + filter_dict = {'hrn': [record['hrn']]} + node_info = self.iotlab_api.GetNodes(filter_dict) + # the node_info is about one node only, but it is formatted + # as a list + record.update(node_info[0]) + logger.debug("IOTLABDRIVER.PY \t \ + fill_record_info NODE" % (record)) + + #If the record is a SFA slice record, then add information + #about the user of this slice. This kind of + #information is in the Iotlab's DB. + if str(record['type']) == 'slice': + if 'reg_researchers' in record and isinstance(record + ['reg_researchers'], + list): + record['reg_researchers'] = \ + record['reg_researchers'][0].__dict__ + record.update( + {'PI': [record['reg_researchers']['hrn']], + 'researcher': [record['reg_researchers']['hrn']], + 'name': record['hrn'], + 'oar_job_id': [], + 'node_ids': [], + 'person_ids': [record['reg_researchers'] + ['record_id']], + # For client_helper.py compatibility + 'geni_urn': '', + # For client_helper.py compatibility + 'keys': '', + # For client_helper.py compatibility + 'key_ids': ''}) + + #Get iotlab slice record and oar job id if any. + recslice_list = self.iotlab_api.GetSlices( + slice_filter=str(record['hrn']), + slice_filter_type='slice_hrn') + + logger.debug("IOTLABDRIVER \tfill_record_info \ + TYPE SLICE RECUSER record['hrn'] %s record['oar_job_id']\ + %s " % (record['hrn'], record['oar_job_id'])) + del record['reg_researchers'] + try: + for rec in recslice_list: + logger.debug("IOTLABDRIVER\r\n \t \ + fill_record_info oar_job_id %s " + % (rec['oar_job_id'])) + + record['node_ids'] = [self.iotlab_api.root_auth + + '.' + hostname for hostname + in rec['node_ids']] + except KeyError: + pass + + logger.debug("IOTLABDRIVER.PY \t fill_record_info SLICE \ + recslice_list %s \r\n \t RECORD %s \r\n \ + \r\n" % (recslice_list, record)) + + if str(record['type']) == 'user': + #The record is a SFA user record. + #Get the information about his slice from Iotlab's DB + #and add it to the user record. + recslice_list = self.iotlab_api.GetSlices( + slice_filter=record['record_id'], + slice_filter_type='record_id_user') + + logger.debug("IOTLABDRIVER.PY \t fill_record_info \ + TYPE USER recslice_list %s \r\n \t RECORD %s \r\n" + % (recslice_list, record)) + #Append slice record in records list, + #therefore fetches user and slice info again(one more loop) + #Will update PIs and researcher for the slice + + recuser = recslice_list[0]['reg_researchers'] + logger.debug("IOTLABDRIVER.PY \t fill_record_info USER \ + recuser %s \r\n \r\n" % (recuser)) + recslice = {} + recslice = recslice_list[0] + recslice.update( + {'PI': [recuser['hrn']], + 'researcher': [recuser['hrn']], + 'name': record['hrn'], + 'node_ids': [], + 'oar_job_id': [], + 'person_ids': [recuser['record_id']]}) + try: + for rec in recslice_list: + recslice['oar_job_id'].append(rec['oar_job_id']) + except KeyError: + pass + + recslice.update({'type': 'slice', + 'hrn': recslice_list[0]['hrn']}) + + #GetPersons takes [] as filters + user_iotlab = self.iotlab_api.GetPersons([record]) + + record.update(user_iotlab[0]) + #For client_helper.py compatibility + record.update( + {'geni_urn': '', + 'keys': '', + 'key_ids': ''}) + record_list.append(recslice) + + logger.debug("IOTLABDRIVER.PY \t \ + fill_record_info ADDING SLICE\ + INFO TO USER records %s" % (record_list)) + + except TypeError, error: + logger.log_exc("IOTLABDRIVER \t fill_record_info EXCEPTION %s" + % (error)) + + return record_list + + def sliver_status(self, slice_urn, slice_hrn): + """ + Receive a status request for slice named urn/hrn + urn:publicid:IDN+iotlab+nturro_slice hrn iotlab.nturro_slice + shall return a structure as described in + http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus + NT : not sure if we should implement this or not, but used by sface. + + :param slice_urn: slice urn + :type slice_urn: string + :param slice_hrn: slice hrn + :type slice_hrn: string + + """ + + #First get the slice with the slice hrn + slice_list = self.iotlab_api.GetSlices(slice_filter=slice_hrn, + slice_filter_type='slice_hrn') + + if len(slice_list) == 0: + raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn)) + + #Used for fetching the user info witch comes along the slice info + one_slice = slice_list[0] + + #Make a list of all the nodes hostnames in use for this slice + slice_nodes_list = [] + slice_nodes_list = one_slice['node_ids'] + #Get all the corresponding nodes details + nodes_all = self.iotlab_api.GetNodes( + {'hostname': slice_nodes_list}, + ['node_id', 'hostname', 'site', 'boot_state']) + nodeall_byhostname = dict([(one_node['hostname'], one_node) + for one_node in nodes_all]) + + for single_slice in slice_list: + #For compatibility + top_level_status = 'empty' + result = {} + result.fromkeys( + ['geni_urn', 'geni_error', 'iotlab_login', 'geni_status', + 'geni_resources'], None) + # result.fromkeys(\ + # ['geni_urn','geni_error', 'pl_login','geni_status', + # 'geni_resources'], None) + # result['pl_login'] = one_slice['reg_researchers'][0].hrn + result['iotlab_login'] = one_slice['user'] + logger.debug("Slabdriver - sliver_status Sliver status \ + urn %s hrn %s single_slice %s \r\n " + % (slice_urn, slice_hrn, single_slice)) + + if 'node_ids' not in single_slice: + #No job in the slice + result['geni_status'] = top_level_status + result['geni_resources'] = [] + return result + + top_level_status = 'ready' + + #A job is running on Iotlab for this slice + # report about the local nodes that are in the slice only + + result['geni_urn'] = slice_urn + + resources = [] + for node_hostname in single_slice['node_ids']: + res = {} + res['iotlab_hostname'] = node_hostname + res['iotlab_boot_state'] = \ + nodeall_byhostname[node_hostname]['boot_state'] + + #res['pl_hostname'] = node['hostname'] + #res['pl_boot_state'] = \ + #nodeall_byhostname[node['hostname']]['boot_state'] + #res['pl_last_contact'] = strftime(self.time_format, \ + #gmtime(float(timestamp))) + sliver_id = Xrn( + slice_urn, type='slice', + id=nodeall_byhostname[node_hostname]['node_id']).urn + + res['geni_urn'] = sliver_id + #node_name = node['hostname'] + if nodeall_byhostname[node_hostname]['boot_state'] == 'Alive': + + res['geni_status'] = 'ready' + else: + res['geni_status'] = 'failed' + top_level_status = 'failed' + + res['geni_error'] = '' + + resources.append(res) + + result['geni_status'] = top_level_status + result['geni_resources'] = resources + logger.debug("IOTLABDRIVER \tsliver_statusresources %s res %s " + % (resources, res)) + return result + + @staticmethod + def get_user_record(hrn): + """ + + Returns the user record based on the hrn from the SFA DB . + + :param hrn: user's hrn + :type hrn: string + :returns: user record from SFA database + :rtype: RegUser + + """ + return dbsession.query(RegRecord).filter_by(hrn=hrn).first() + + def testbed_name(self): + """ + + Returns testbed's name. + :returns: testbed authority name. + :rtype: string + + """ + return self.hrn + + # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory + def aggregate_version(self): + """ + + Returns the testbed's supported rspec advertisement and request + versions. + :returns: rspec versions supported ad a dictionary. + :rtype: dict + + """ + version_manager = VersionManager() + ad_rspec_versions = [] + request_rspec_versions = [] + for rspec_version in version_manager.versions: + if rspec_version.content_type in ['*', 'ad']: + ad_rspec_versions.append(rspec_version.to_dict()) + if rspec_version.content_type in ['*', 'request']: + request_rspec_versions.append(rspec_version.to_dict()) + return { + 'testbed': self.testbed_name(), + 'geni_request_rspec_versions': request_rspec_versions, + 'geni_ad_rspec_versions': ad_rspec_versions} + + def _get_requested_leases_list(self, rspec): + """ + Process leases in rspec depending on the rspec version (format) + type. Find the lease requests in the rspec and creates + a lease request list with the mandatory information ( nodes, + start time and duration) of the valid leases (duration above or + equal to the iotlab experiment minimum duration). + + :param rspec: rspec request received. + :type rspec: RSpec + :returns: list of lease requests found in the rspec + :rtype: list + """ + requested_lease_list = [] + for lease in rspec.version.get_leases(): + single_requested_lease = {} + logger.debug("IOTLABDRIVER.PY \t \ + _get_requested_leases_list lease %s " % (lease)) + + if not lease.get('lease_id'): + if get_authority(lease['component_id']) == \ + self.iotlab_api.root_auth: + single_requested_lease['hostname'] = \ + iotlab_xrn_to_hostname(\ + lease.get('component_id').strip()) + single_requested_lease['start_time'] = \ + lease.get('start_time') + single_requested_lease['duration'] = lease.get('duration') + #Check the experiment's duration is valid before adding + #the lease to the requested leases list + duration_in_seconds = \ + int(single_requested_lease['duration']) + if duration_in_seconds >= self.iotlab_api.GetMinExperimentDurationInGranularity(): + requested_lease_list.append(single_requested_lease) + + return requested_lease_list + + @staticmethod + def _group_leases_by_start_time(requested_lease_list): + """ + Create dict of leases by start_time, regrouping nodes reserved + at the same time, for the same amount of time so as to + define one job on OAR. + + :param requested_lease_list: list of leases + :type requested_lease_list: list + :returns: Dictionary with key = start time, value = list of leases + with the same start time. + :rtype: dictionary + + """ + + requested_job_dict = {} + for lease in requested_lease_list: + + #In case it is an asap experiment start_time is empty + if lease['start_time'] == '': + lease['start_time'] = '0' + + if lease['start_time'] not in requested_job_dict: + if isinstance(lease['hostname'], str): + lease['hostname'] = [lease['hostname']] + + requested_job_dict[lease['start_time']] = lease + + else: + job_lease = requested_job_dict[lease['start_time']] + if lease['duration'] == job_lease['duration']: + job_lease['hostname'].append(lease['hostname']) + + return requested_job_dict + + def _process_requested_jobs(self, rspec): + """ + Turns the requested leases and information into a dictionary + of requested jobs, grouped by starting time. + + :param rspec: RSpec received + :type rspec : RSpec + :rtype: dictionary + + """ + requested_lease_list = self._get_requested_leases_list(rspec) + logger.debug("IOTLABDRIVER _process_requested_jobs \ + requested_lease_list %s" % (requested_lease_list)) + job_dict = self._group_leases_by_start_time(requested_lease_list) + logger.debug("IOTLABDRIVER _process_requested_jobs job_dict\ + %s" % (job_dict)) + + return job_dict + + def create_sliver(self, slice_urn, slice_hrn, creds, rspec_string, + users, options): + """Answer to CreateSliver. + + Creates the leases and slivers for the users from the information + found in the rspec string. + Launch experiment on OAR if the requested leases is valid. Delete + no longer requested leases. + + + :param creds: user's credentials + :type creds: string + :param users: user record list + :type users: list + :param options: + :type options: + + :returns: a valid Rspec for the slice which has just been + modified. + :rtype: RSpec + + + """ + aggregate = IotlabAggregate(self) + + slices = IotlabSlices(self) + peer = slices.get_peer(slice_hrn) + sfa_peer = slices.get_sfa_peer(slice_hrn) + slice_record = None + + if not isinstance(creds, list): + creds = [creds] + + if users: + slice_record = users[0].get('slice_record', {}) + logger.debug("IOTLABDRIVER.PY \t ===============create_sliver \t\ + creds %s \r\n \r\n users %s" + % (creds, users)) + slice_record['user'] = {'keys': users[0]['keys'], + 'email': users[0]['email'], + 'hrn': slice_record['reg-researchers'][0]} + # parse rspec + rspec = RSpec(rspec_string) + logger.debug("IOTLABDRIVER.PY \t create_sliver \trspec.version \ + %s slice_record %s users %s" + % (rspec.version, slice_record, users)) + + # ensure site record exists? + # ensure slice record exists + #Removed options in verify_slice SA 14/08/12 + #Removed peer record in verify_slice SA 18/07/13 + sfa_slice = slices.verify_slice(slice_hrn, slice_record, sfa_peer) + + # ensure person records exists + #verify_persons returns added persons but the return value + #is not used + #Removed peer record and sfa_peer in verify_persons SA 18/07/13 + slices.verify_persons(slice_hrn, sfa_slice, users, options=options) + #requested_attributes returned by rspec.version.get_slice_attributes() + #unused, removed SA 13/08/12 + #rspec.version.get_slice_attributes() + + logger.debug("IOTLABDRIVER.PY create_sliver slice %s " % (sfa_slice)) + + # add/remove slice from nodes + + #requested_slivers = [node.get('component_id') \ + #for node in rspec.version.get_nodes_with_slivers()\ + #if node.get('authority_id') is self.iotlab_api.root_auth] + #l = [ node for node in rspec.version.get_nodes_with_slivers() ] + #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \ + #requested_slivers %s listnodes %s" \ + #%(requested_slivers,l)) + #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12. + #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer) + + requested_job_dict = self._process_requested_jobs(rspec) + + logger.debug("IOTLABDRIVER.PY \tcreate_sliver requested_job_dict %s " + % (requested_job_dict)) + #verify_slice_leases returns the leases , but the return value is unused + #here. Removed SA 13/08/12 + slices.verify_slice_leases(sfa_slice, + requested_job_dict, peer) + + return aggregate.get_rspec(slice_xrn=slice_urn, + login=sfa_slice['login'], + version=rspec.version) + + def delete_sliver(self, slice_urn, slice_hrn, creds, options): + """ + Deletes the lease associated with the slice hrn and the credentials + if the slice belongs to iotlab. Answer to DeleteSliver. + + :param slice_urn: urn of the slice + :param slice_hrn: name of the slice + :param creds: slice credenials + :type slice_urn: string + :type slice_hrn: string + :type creds: ? unused + + :returns: 1 if the slice to delete was not found on iotlab, + True if the deletion was successful, False otherwise otherwise. + + .. note:: Should really be named delete_leases because iotlab does + not have any slivers, but only deals with leases. However, + SFA api only have delete_sliver define so far. SA 13/05/2013 + .. note:: creds are unused, and are not used either in the dummy driver + delete_sliver . + """ + + sfa_slice_list = self.iotlab_api.GetSlices( + slice_filter=slice_hrn, + slice_filter_type='slice_hrn') + + if not sfa_slice_list: + return 1 + + #Delete all leases in the slice + for sfa_slice in sfa_slice_list: + logger.debug("IOTLABDRIVER.PY delete_sliver slice %s" % (sfa_slice)) + slices = IotlabSlices(self) + # determine if this is a peer slice + + peer = slices.get_peer(slice_hrn) + + logger.debug("IOTLABDRIVER.PY delete_sliver peer %s \ + \r\n \t sfa_slice %s " % (peer, sfa_slice)) + try: + self.iotlab_api.DeleteSliceFromNodes(sfa_slice) + return True + except: + return False + + def list_resources (self, slice_urn, slice_hrn, creds, options): + """ + + List resources from the iotlab aggregate and returns a Rspec + advertisement with resources found when slice_urn and slice_hrn are + None (in case of resource discovery). + If a slice hrn and urn are provided, list experiment's slice + nodes in a rspec format. Answer to ListResources. + Caching unused. + + :param slice_urn: urn of the slice + :param slice_hrn: name of the slice + :param creds: slice credenials + :type slice_urn: string + :type slice_hrn: string + :type creds: ? unused + :param options: options used when listing resources (list_leases, info, + geni_available) + :returns: rspec string in xml + :rtype: string + + .. note:: creds are unused + """ + + #cached_requested = options.get('cached', True) + + version_manager = VersionManager() + # get the rspec's return format from options + rspec_version = \ + version_manager.get_version(options.get('geni_rspec_version')) + version_string = "rspec_%s" % (rspec_version) + + #panos adding the info option to the caching key (can be improved) + if options.get('info'): + version_string = version_string + "_" + \ + options.get('info', 'default') + + # Adding the list_leases option to the caching key + if options.get('list_leases'): + version_string = version_string + "_" + \ + options.get('list_leases', 'default') + + # Adding geni_available to caching key + if options.get('geni_available'): + version_string = version_string + "_" + \ + str(options.get('geni_available')) + + # look in cache first + #if cached_requested and self.cache and not slice_hrn: + #rspec = self.cache.get(version_string) + #if rspec: + #logger.debug("IotlabDriver.ListResources: \ + #returning cached advertisement") + #return rspec + + #panos: passing user-defined options + aggregate = IotlabAggregate(self) + + rspec = aggregate.get_rspec(slice_xrn=slice_urn, + version=rspec_version, options=options) + + # cache the result + #if self.cache and not slice_hrn: + #logger.debug("Iotlab.ListResources: stores advertisement in cache") + #self.cache.add(version_string, rspec) + + return rspec + + + def list_slices(self, creds, options): + """Answer to ListSlices. + + List slices belonging to iotlab, returns slice urns list. + No caching used. Options unused but are defined in the SFA method + api prototype. + + :returns: slice urns list + :rtype: list + + .. note:: creds are unused + """ + # look in cache first + #if self.cache: + #slices = self.cache.get('slices') + #if slices: + #logger.debug("PlDriver.list_slices returns from cache") + #return slices + + # get data from db + + slices = self.iotlab_api.GetSlices() + logger.debug("IOTLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" + % (slices)) + slice_hrns = [iotlab_slice['hrn'] for iotlab_slice in slices] + + slice_urns = [hrn_to_urn(slice_hrn, 'slice') + for slice_hrn in slice_hrns] + + # cache the result + #if self.cache: + #logger.debug ("IotlabDriver.list_slices stores value in cache") + #self.cache.add('slices', slice_urns) + + return slice_urns + + + def register(self, sfa_record, hrn, pub_key): + """ + Adding new user, slice, node or site should not be handled + by SFA. + + ..warnings:: should not be used. Different components are in charge of + doing this task. Adding nodes = OAR + Adding users = LDAP Iotlab + Adding slice = Import from LDAP users + Adding site = OAR + + :param sfa_record: record provided by the client of the + Register API call. + :type sfa_record: dict + :param pub_key: public key of the user + :type pub_key: string + + .. note:: DOES NOTHING. Returns -1. + + """ + return -1 + + + def update(self, old_sfa_record, new_sfa_record, hrn, new_key): + """ + No site or node record update allowed in Iotlab. The only modifications + authorized here are key deletion/addition on an existing user and + password change. On an existing user, CAN NOT BE MODIFIED: 'first_name', + 'last_name', 'email'. DOES NOT EXIST IN SENSLAB: 'phone', 'url', 'bio', + 'title', 'accepted_aup'. A slice is bound to its user, so modifying the + user's ssh key should nmodify the slice's GID after an import procedure. + + :param old_sfa_record: what is in the db for this hrn + :param new_sfa_record: what was passed to the update call + :param new_key: the new user's public key + :param hrn: the user's sfa hrn + :type old_sfa_record: dict + :type new_sfa_record: dict + :type new_key: string + :type hrn: string + + TODO: needs review + .. seealso:: update in driver.py. + + """ + pointer = old_sfa_record['pointer'] + old_sfa_record_type = old_sfa_record['type'] + + # new_key implemented for users only + if new_key and old_sfa_record_type not in ['user']: + raise UnknownSfaType(old_sfa_record_type) + + if old_sfa_record_type == "user": + update_fields = {} + all_fields = new_sfa_record + for key in all_fields.keys(): + if key in ['key', 'password']: + update_fields[key] = all_fields[key] + + if new_key: + # must check this key against the previous one if it exists + persons = self.iotlab_api.GetPersons([old_sfa_record]) + person = persons[0] + keys = [person['pkey']] + #Get all the person's keys + keys_dict = self.iotlab_api.GetKeys(keys) + + # Delete all stale keys, meaning the user has only one key + #at a time + #TODO: do we really want to delete all the other keys? + #Is this a problem with the GID generation to have multiple + #keys? SA 30/05/13 + key_exists = False + if key in keys_dict: + key_exists = True + else: + #remove all the other keys + for key in keys_dict: + self.iotlab_api.DeleteKey(person, key) + self.iotlab_api.AddPersonKey( + person, {'sshPublicKey': person['pkey']}, + {'sshPublicKey': new_key}) + return True + + def remove(self, sfa_record): + """ + + Removes users only. Mark the user as disabled in LDAP. The user and his + slice are then deleted from the db by running an import on the registry. + + :param sfa_record: record is the existing sfa record in the db + :type sfa_record: dict + + ..warning::As fas as the slice is concerned, here only the leases are + removed from the slice. The slice is record itself is not removed + from the db. + + TODO: needs review + + TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013, + + TODO: return boolean for the slice part + """ + sfa_record_type = sfa_record['type'] + hrn = sfa_record['hrn'] + if sfa_record_type == 'user': + + #get user from iotlab ldap + person = self.iotlab_api.GetPersons(sfa_record) + #No registering at a given site in Iotlab. + #Once registered to the LDAP, all iotlab sites are + #accesible. + if person: + #Mark account as disabled in ldap + return self.iotlab_api.DeletePerson(sfa_record) + + elif sfa_record_type == 'slice': + if self.iotlab_api.GetSlices(slice_filter=hrn, + slice_filter_type='slice_hrn'): + ret = self.iotlab_api.DeleteSlice(sfa_record) + return True diff --git a/sfa/cortexlab/cortexlabpostgres.py b/sfa/cortexlab/cortexlabpostgres.py new file mode 100644 index 00000000..cd4fc585 --- /dev/null +++ b/sfa/cortexlab/cortexlabpostgres.py @@ -0,0 +1,264 @@ +""" +File defining classes to handle the table in the iotlab dedicated database. +""" + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +# from sfa.util.config import Config +from sfa.util.sfalogging import logger + +from sqlalchemy import Column, Integer, String +from sqlalchemy import Table, MetaData +from sqlalchemy.ext.declarative import declarative_base + +# from sqlalchemy.dialects import postgresql + +from sqlalchemy.exc import NoSuchTableError + + +#Dict holding the columns names of the table as keys +#and their type, used for creation of the table +slice_table = {'record_id_user': 'integer PRIMARY KEY references X ON DELETE \ + CASCADE ON UPDATE CASCADE', 'oar_job_id': 'integer DEFAULT -1', + 'record_id_slice': 'integer', 'slice_hrn': 'text NOT NULL'} + +#Dict with all the specific iotlab tables +tablenames_dict = {'iotlab_xp': slice_table} + + +IotlabBase = declarative_base() + + +class IotlabXP (IotlabBase): + """ SQL alchemy class to manipulate the rows of the slice_iotlab table in + iotlab_sfa database. Handles the records representation and creates the + table if it does not exist yet. + + """ + __tablename__ = 'iotlab_xp' + + slice_hrn = Column(String) + job_id = Column(Integer, primary_key=True) + end_time = Column(Integer, nullable=False) + + def __init__(self, slice_hrn=None, job_id=None, end_time=None): + """ + Defines a row of the slice_iotlab table + """ + if slice_hrn: + self.slice_hrn = slice_hrn + if job_id: + self.job_id = job_id + if end_time: + self.end_time = end_time + + def __repr__(self): + """Prints the SQLAlchemy record to the format defined + by the function. + """ + result = " 0: + self.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') + self.iotlab_session.commit() + return + + def __init__(self, config, debug=False): + self.sl_base = IotlabBase + + # Check whether we already have an instance + if IotlabDB._connection_singleton is None: + IotlabDB._connection_singleton = IotlabDB.Singleton(config, debug) + + # Store instance reference as the only member in the handle + self._EventHandler_singleton = IotlabDB._connection_singleton + + def __getattr__(self, aAttr): + """ + Delegate access to implementation. + + :param aAttr: Attribute wanted. + :returns: Attribute + """ + return getattr(self._connection_singleton, aAttr) + + + + # def __setattr__(self, aAttr, aValue): + # """Delegate access to implementation. + + # :param attr: Attribute wanted. + # :param value: Vaule to be set. + # :return: Result of operation. + # """ + # return setattr(self._connection_singleton, aAttr, aValue) + + def exists(self, tablename): + """ + Checks if the table specified as tablename exists. + :param tablename: name of the table in the db that has to be checked. + :type tablename: string + :returns: True if the table exists, False otherwise. + :rtype: bool + + """ + metadata = MetaData(bind=self.iotlab_engine) + try: + table = Table(tablename, metadata, autoload=True) + return True + + except NoSuchTableError: + logger.log_exc("IOTLABPOSTGRES tablename %s does not exist" + % (tablename)) + return False + + def createtable(self): + """ + Creates all the table sof the engine. + Uses the global dictionnary holding the tablenames and the table schema. + + """ + + logger.debug("IOTLABPOSTGRES createtable \ + IotlabBase.metadata.sorted_tables %s \r\n engine %s" + % (IotlabBase.metadata.sorted_tables, self.iotlab_engine)) + IotlabBase.metadata.create_all(self.iotlab_engine) + return diff --git a/sfa/cortexlab/cortexlabslices.py b/sfa/cortexlab/cortexlabslices.py new file mode 100644 index 00000000..2ef77154 --- /dev/null +++ b/sfa/cortexlab/cortexlabslices.py @@ -0,0 +1,587 @@ +""" +This file defines the IotlabSlices class by which all the slice checkings +upon lease creation are done. +""" +from sfa.util.xrn import get_authority, urn_to_hrn +from sfa.util.sfalogging import logger + +MAXINT = 2L**31-1 + + +class IotlabSlices: + """ + This class is responsible for checking the slice when creating a + lease or a sliver. Those checks include verifying that the user is valid, + that the slice is known from the testbed or from our peers, that the list + of nodes involved has not changed (in this case the lease is modified + accordingly). + """ + rspec_to_slice_tag = {'max_rate': 'net_max_rate'} + + def __init__(self, driver): + """ + Get the reference to the driver here. + """ + self.driver = driver + + def get_peer(self, xrn): + """ + Finds the authority of a resource based on its xrn. + If the authority is Iotlab (local) return None, + Otherwise, look up in the DB if Iotlab is federated with this site + authority and returns its DB record if it is the case. + + :param xrn: resource's xrn + :type xrn: string + :returns: peer record + :rtype: dict + + """ + hrn, hrn_type = urn_to_hrn(xrn) + #Does this slice belong to a local site or a peer iotlab site? + peer = None + + # get this slice's authority (site) + slice_authority = get_authority(hrn) + #Iotlab stuff + #This slice belongs to the current site + if slice_authority == self.driver.iotlab_api.root_auth: + site_authority = slice_authority + return None + + site_authority = get_authority(slice_authority).lower() + # get this site's authority (sfa root authority or sub authority) + + logger.debug("IOTLABSLICES \t get_peer slice_authority %s \ + site_authority %s hrn %s" + % (slice_authority, site_authority, hrn)) + + # check if we are already peered with this site_authority + #if so find the peer record + peers = self.driver.iotlab_api.GetPeers(peer_filter=site_authority) + for peer_record in peers: + if site_authority == peer_record.hrn: + peer = peer_record + logger.debug(" IOTLABSLICES \tget_peer peer %s " % (peer)) + return peer + + def get_sfa_peer(self, xrn): + """Returns the authority name for the xrn or None if the local site + is the authority. + + :param xrn: the xrn of the resource we are looking the authority for. + :type xrn: string + :returns: the resources's authority name. + :rtype: string + + """ + hrn, hrn_type = urn_to_hrn(xrn) + + # return the authority for this hrn or None if we are the authority + sfa_peer = None + slice_authority = get_authority(hrn) + site_authority = get_authority(slice_authority) + + if site_authority != self.driver.hrn: + sfa_peer = site_authority + + return sfa_peer + + def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer): + """ + Compare requested leases with the leases already scheduled/ + running in OAR. If necessary, delete and recreate modified leases, + and delete no longer requested ones. + + :param sfa_slice: sfa slice record + :param requested_jobs_dict: dictionary of requested leases + :param peer: sfa peer record + + :type sfa_slice: dict + :type requested_jobs_dict: dict + :type peer: dict + :returns: leases list of dictionary + :rtype: list + + """ + + logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s " + % (sfa_slice)) + #First get the list of current leases from OAR + leases = self.driver.iotlab_api.GetLeases({'slice_hrn': sfa_slice['hrn']}) + logger.debug("IOTLABSLICES verify_slice_leases requested_jobs_dict %s \ + leases %s " % (requested_jobs_dict, leases)) + + current_nodes_reserved_by_start_time = {} + requested_nodes_by_start_time = {} + leases_by_start_time = {} + reschedule_jobs_dict = {} + + #Create reduced dictionary with key start_time and value + # the list of nodes + #-for the leases already registered by OAR first + # then for the new leases requested by the user + + #Leases already scheduled/running in OAR + for lease in leases: + current_nodes_reserved_by_start_time[lease['t_from']] = \ + lease['reserved_nodes'] + leases_by_start_time[lease['t_from']] = lease + + #First remove job whose duration is too short + for job in requested_jobs_dict.values(): + job['duration'] = \ + str(int(job['duration']) \ + * self.driver.iotlab_api.GetLeaseGranularity()) + if job['duration'] < self.driver.iotlab_api.GetLeaseGranularity(): + del requested_jobs_dict[job['start_time']] + + #Requested jobs + for start_time in requested_jobs_dict: + requested_nodes_by_start_time[int(start_time)] = \ + requested_jobs_dict[start_time]['hostname'] + #Check if there is any difference between the leases already + #registered in OAR and the requested jobs. + #Difference could be: + #-Lease deleted in the requested jobs + #-Added/removed nodes + #-Newly added lease + + logger.debug("IOTLABSLICES verify_slice_leases \ + requested_nodes_by_start_time %s \ + "% (requested_nodes_by_start_time)) + #Find all deleted leases + start_time_list = \ + list(set(leases_by_start_time.keys()).\ + difference(requested_nodes_by_start_time.keys())) + deleted_leases = [leases_by_start_time[start_time]['lease_id'] \ + for start_time in start_time_list] + + + #Find added or removed nodes in exisiting leases + for start_time in requested_nodes_by_start_time: + logger.debug("IOTLABSLICES verify_slice_leases start_time %s \ + "%( start_time)) + if start_time in current_nodes_reserved_by_start_time: + + if requested_nodes_by_start_time[start_time] == \ + current_nodes_reserved_by_start_time[start_time]: + continue + + else: + update_node_set = \ + set(requested_nodes_by_start_time[start_time]) + added_nodes = \ + update_node_set.difference(\ + current_nodes_reserved_by_start_time[start_time]) + shared_nodes = \ + update_node_set.intersection(\ + current_nodes_reserved_by_start_time[start_time]) + old_nodes_set = \ + set(\ + current_nodes_reserved_by_start_time[start_time]) + removed_nodes = \ + old_nodes_set.difference(\ + requested_nodes_by_start_time[start_time]) + logger.debug("IOTLABSLICES verify_slice_leases \ + shared_nodes %s added_nodes %s removed_nodes %s"\ + %(shared_nodes, added_nodes,removed_nodes )) + #If the lease is modified, delete it before + #creating it again. + #Add the deleted lease job id in the list + #WARNING :rescheduling does not work if there is already + # 2 running/scheduled jobs because deleting a job + #takes time SA 18/10/2012 + if added_nodes or removed_nodes: + deleted_leases.append(\ + leases_by_start_time[start_time]['lease_id']) + #Reschedule the job + if added_nodes or shared_nodes: + reschedule_jobs_dict[str(start_time)] = \ + requested_jobs_dict[str(start_time)] + + else: + #New lease + + job = requested_jobs_dict[str(start_time)] + logger.debug("IOTLABSLICES \ + NEWLEASE slice %s job %s" + % (sfa_slice, job)) + self.driver.iotlab_api.AddLeases( + job['hostname'], + sfa_slice, int(job['start_time']), + int(job['duration'])) + + #Deleted leases are the ones with lease id not declared in the Rspec + if deleted_leases: + self.driver.iotlab_api.DeleteLeases(deleted_leases, + sfa_slice['user']['uid']) + logger.debug("IOTLABSLICES \ + verify_slice_leases slice %s deleted_leases %s" + % (sfa_slice, deleted_leases)) + + if reschedule_jobs_dict: + for start_time in reschedule_jobs_dict: + job = reschedule_jobs_dict[start_time] + self.driver.iotlab_api.AddLeases( + job['hostname'], + sfa_slice, int(job['start_time']), + int(job['duration'])) + return leases + + def verify_slice_nodes(self, sfa_slice, requested_slivers, peer): + """Check for wanted and unwanted nodes in the slice. + + Removes nodes and associated leases that the user does not want anymore + by deleteing the associated job in OAR (DeleteSliceFromNodes). + Returns the nodes' hostnames that are going to be in the slice. + + :param sfa_slice: slice record. Must contain node_ids and list_node_ids. + + :param requested_slivers: list of requested nodes' hostnames. + :param peer: unused so far. + + :type sfa_slice: dict + :type requested_slivers: list + :type peer: string + + :returns: list requested nodes hostnames + :rtype: list + + .. warning:: UNUSED SQA 24/07/13 + .. seealso:: DeleteSliceFromNodes + .. todo:: check what to do with the peer? Can not remove peer nodes from + slice here. Anyway, in this case, the peer should have gotten the + remove request too. + + """ + current_slivers = [] + deleted_nodes = [] + + if 'node_ids' in sfa_slice: + nodes = self.driver.iotlab_api.GetNodes( + sfa_slice['list_node_ids'], + ['hostname']) + current_slivers = [node['hostname'] for node in nodes] + + # remove nodes not in rspec + deleted_nodes = list(set(current_slivers). + difference(requested_slivers)) + + logger.debug("IOTLABSLICES \tverify_slice_nodes slice %s\ + \r\n \r\n deleted_nodes %s" + % (sfa_slice, deleted_nodes)) + + if deleted_nodes: + #Delete the entire experience + self.driver.iotlab_api.DeleteSliceFromNodes(sfa_slice) + return nodes + + def verify_slice(self, slice_hrn, slice_record, sfa_peer): + """Ensures slice record exists. + + The slice record must exist either in Iotlab or in the other + federated testbed (sfa_peer). If the slice does not belong to Iotlab, + check if the user already exists in LDAP. In this case, adds the slice + to the sfa DB and associates its LDAP user. + + :param slice_hrn: slice's name + :param slice_record: sfa record of the slice + :param sfa_peer: name of the peer authority if any.(not Iotlab). + + :type slice_hrn: string + :type slice_record: dictionary + :type sfa_peer: string + + .. seealso:: AddSlice + + + """ + + slicename = slice_hrn + # check if slice belongs to Iotlab + slices_list = self.driver.iotlab_api.GetSlices( + slice_filter=slicename, slice_filter_type='slice_hrn') + + sfa_slice = None + + if slices_list: + for sl in slices_list: + + logger.debug("IOTLABSLICES \t verify_slice slicename %s \ + slices_list %s sl %s \r slice_record %s" + % (slicename, slices_list, sl, slice_record)) + sfa_slice = sl + sfa_slice.update(slice_record) + + else: + #Search for user in ldap based on email SA 14/11/12 + ldap_user = self.driver.iotlab_api.ldap.LdapFindUser(\ + slice_record['user']) + logger.debug(" IOTLABSLICES \tverify_slice Oups \ + slice_record %s sfa_peer %s ldap_user %s" + % (slice_record, sfa_peer, ldap_user)) + #User already registered in ldap, meaning user should be in SFA db + #and hrn = sfa_auth+ uid + sfa_slice = {'hrn': slicename, + 'node_list': [], + 'authority': slice_record['authority'], + 'gid': slice_record['gid'], + 'slice_id': slice_record['record_id'], + 'reg-researchers': slice_record['reg-researchers'], + 'peer_authority': str(sfa_peer) + } + + if ldap_user: + hrn = self.driver.iotlab_api.root_auth + '.' + ldap_user['uid'] + user = self.driver.get_user_record(hrn) + + logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s" + % (hrn, user)) + + # add the external slice to the local SFA iotlab DB + if sfa_slice: + self.driver.iotlab_api.AddSlice(sfa_slice, user) + + logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK") + return sfa_slice + + + def verify_persons(self, slice_hrn, slice_record, users, options={}): + """Ensures the users in users list exist and are enabled in LDAP. Adds + person if needed. + + Checking that a user exist is based on the user's email. If the user is + still not found in the LDAP, it means that the user comes from another + federated testbed. In this case an account has to be created in LDAP + so as to enable the user to use the testbed, since we trust the testbed + he comes from. This is done by calling AddPerson. + + :param slice_hrn: slice name + :param slice_record: record of the slice_hrn + :param users: users is a record list. Records can either be + local records or users records from known and trusted federated + sites.If the user is from another site that iotlab doesn't trust yet, + then Resolve will raise an error before getting to create_sliver. + + :type slice_hrn: string + :type slice_record: string + :type users: list + + .. seealso:: AddPerson + .. note:: Removed unused peer and sfa_peer parameters. SA 18/07/13. + + + """ + #TODO SA 21/08/12 verify_persons Needs review + + logger.debug("IOTLABSLICES \tverify_persons \tslice_hrn %s \ + \t slice_record %s\r\n users %s \t " + % (slice_hrn, slice_record, users)) + users_by_id = {} + + users_by_email = {} + #users_dict : dict whose keys can either be the user's hrn or its id. + #Values contains only id and hrn + users_dict = {} + + #First create dicts by hrn and id for each user in the user record list: + for info in users: + if 'slice_record' in info: + slice_rec = info['slice_record'] + if 'user' in slice_rec : + user = slice_rec['user'] + + if 'email' in user: + users_by_email[user['email']] = user + users_dict[user['email']] = user + + logger.debug("IOTLABSLICES.PY \t verify_person \ + users_dict %s \r\n user_by_email %s \r\n \ + \tusers_by_id %s " + % (users_dict, users_by_email, users_by_id)) + + existing_user_ids = [] + existing_user_emails = [] + existing_users = [] + # Check if user is in Iotlab LDAP using its hrn. + # Assuming Iotlab is centralised : one LDAP for all sites, + # user's record_id unknown from LDAP + # LDAP does not provide users id, therefore we rely on email to find the + # user in LDAP + + if users_by_email: + #Construct the list of filters (list of dicts) for GetPersons + filter_user = [users_by_email[email] for email in users_by_email] + #Check user i in LDAP with GetPersons + #Needed because what if the user has been deleted in LDAP but + #is still in SFA? + existing_users = self.driver.iotlab_api.GetPersons(filter_user) + logger.debug(" \r\n IOTLABSLICES.PY \tverify_person filter_user \ + %s existing_users %s " + % (filter_user, existing_users)) + #User is in iotlab LDAP + if existing_users: + for user in existing_users: + users_dict[user['email']].update(user) + existing_user_emails.append( + users_dict[user['email']]['email']) + + + # User from another known trusted federated site. Check + # if a iotlab account matching the email has already been created. + else: + req = 'mail=' + if isinstance(users, list): + req += users[0]['email'] + else: + req += users['email'] + ldap_reslt = self.driver.iotlab_api.ldap.LdapSearch(req) + + if ldap_reslt: + logger.debug(" IOTLABSLICES.PY \tverify_person users \ + USER already in Iotlab \t ldap_reslt %s \ + " % (ldap_reslt)) + existing_users.append(ldap_reslt[1]) + + else: + #User not existing in LDAP + logger.debug("IOTLABSLICES.PY \tverify_person users \ + not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \ + ldap_reslt %s " % (users, ldap_reslt)) + + requested_user_emails = users_by_email.keys() + requested_user_hrns = \ + [users_by_email[user]['hrn'] for user in users_by_email] + logger.debug("IOTLABSLICES.PY \tverify_person \ + users_by_email %s " % (users_by_email)) + + #Check that the user of the slice in the slice record + #matches one of the existing users + try: + if slice_record['PI'][0] in requested_user_hrns: + logger.debug(" IOTLABSLICES \tverify_person ['PI']\ + slice_record %s" % (slice_record)) + + except KeyError: + pass + + # users to be added, removed or updated + #One user in one iotlab slice : there should be no need + #to remove/ add any user from/to a slice. + #However a user from SFA which is not registered in Iotlab yet + #should be added to the LDAP. + added_user_emails = set(requested_user_emails).\ + difference(set(existing_user_emails)) + + + #self.verify_keys(existing_slice_users, updated_users_list, \ + #peer, append) + + added_persons = [] + # add new users + #requested_user_email is in existing_user_emails + if len(added_user_emails) == 0: + slice_record['login'] = users_dict[requested_user_emails[0]]['uid'] + logger.debug(" IOTLABSLICES \tverify_person QUICK DIRTY %s" + % (slice_record)) + + for added_user_email in added_user_emails: + added_user = users_dict[added_user_email] + logger.debug(" IOTLABSLICES \r\n \r\n \t verify_person \ + added_user %s" % (added_user)) + person = {} + person['peer_person_id'] = None + k_list = ['first_name', 'last_name', 'person_id'] + for k in k_list: + if k in added_user: + person[k] = added_user[k] + + person['pkey'] = added_user['keys'][0] + person['mail'] = added_user['email'] + person['email'] = added_user['email'] + person['key_ids'] = added_user.get('key_ids', []) + + ret = self.driver.iotlab_api.AddPerson(person) + if 'uid' in ret: + # meaning bool is True and the AddPerson was successful + person['uid'] = ret['uid'] + slice_record['login'] = person['uid'] + else: + # error message in ret + logger.debug(" IOTLABSLICES ret message %s" %(ret)) + + logger.debug(" IOTLABSLICES \r\n \r\n \t THE SECOND verify_person\ + person %s" % (person)) + #Update slice_Record with the id now known to LDAP + + + added_persons.append(person) + return added_persons + + + def verify_keys(self, persons, users, peer, options={}): + """ + .. warning:: unused + """ + # existing keys + key_ids = [] + for person in persons: + key_ids.extend(person['key_ids']) + keylist = self.driver.iotlab_api.GetKeys(key_ids, ['key_id', 'key']) + + keydict = {} + for key in keylist: + keydict[key['key']] = key['key_id'] + existing_keys = keydict.keys() + + persondict = {} + for person in persons: + persondict[person['email']] = person + + # add new keys + requested_keys = [] + updated_persons = [] + users_by_key_string = {} + for user in users: + user_keys = user.get('keys', []) + updated_persons.append(user) + for key_string in user_keys: + users_by_key_string[key_string] = user + requested_keys.append(key_string) + if key_string not in existing_keys: + key = {'key': key_string, 'key_type': 'ssh'} + #try: + ##if peer: + #person = persondict[user['email']] + #self.driver.iotlab_api.UnBindObjectFromPeer( + # 'person',person['person_id'], + # peer['shortname']) + ret = self.driver.iotlab_api.AddPersonKey( + user['email'], key) + #if peer: + #key_index = user_keys.index(key['key']) + #remote_key_id = user['key_ids'][key_index] + #self.driver.iotlab_api.BindObjectToPeer('key', \ + #key['key_id'], peer['shortname'], \ + #remote_key_id) + + #finally: + #if peer: + #self.driver.iotlab_api.BindObjectToPeer('person', \ + #person['person_id'], peer['shortname'], \ + #user['person_id']) + + # remove old keys (only if we are not appending) + append = options.get('append', True) + if append is False: + removed_keys = set(existing_keys).difference(requested_keys) + for key in removed_keys: + #if peer: + #self.driver.iotlab_api.UnBindObjectFromPeer('key', \ + #key, peer['shortname']) + + user = users_by_key_string[key] + self.driver.iotlab_api.DeleteKey(user, key) + + return diff --git a/sfa/generic/slab.py b/sfa/generic/iotlab.py similarity index 65% rename from sfa/generic/slab.py rename to sfa/generic/iotlab.py index e4a474be..d5636a7c 100644 --- a/sfa/generic/slab.py +++ b/sfa/generic/iotlab.py @@ -4,41 +4,39 @@ import sfa.server.sfaapi -class slab (Generic): - +class iotlab (Generic): + # use the standard api class def api_class (self): return sfa.server.sfaapi.SfaApi - + # the importer class - def importer_class (self): - import sfa.importer.slabimporter - return sfa.importer.slabimporter.SlabImporter - + def importer_class (self): + import sfa.importer.iotlabimporter + return sfa.importer.iotlabimporter.IotlabImporter + # the manager classes for the server-side services def registry_manager_class (self) : - import sfa.managers.registry_manager + import sfa.managers.registry_manager return sfa.managers.registry_manager.RegistryManager - + def slicemgr_manager_class (self) : - import sfa.managers.slice_manager + import sfa.managers.slice_manager return sfa.managers.slice_manager.SliceManager - + def aggregate_manager_class (self) : import sfa.managers.aggregate_manager return sfa.managers.aggregate_manager.AggregateManager # driver class for server-side services, talk to the whole testbed def driver_class (self): - import sfa.managers.v2_to_v3_adapter - return sfa.managers.v2_to_v3_adapter.V2ToV3Adapter + import sfa.iotlab.iotlabdriver + return sfa.iotlab.iotlabdriver.IotlabDriver - # slab does not have a component manager yet + # iotlab does not have a component manager yet # manager class def component_manager_class (self): return None # driver_class def component_driver_class (self): - return None - - + return None \ No newline at end of file diff --git a/sfa/importer/iotlabimporter.py b/sfa/importer/iotlabimporter.py new file mode 100644 index 00000000..86874375 --- /dev/null +++ b/sfa/importer/iotlabimporter.py @@ -0,0 +1,558 @@ +""" File defining the importer class and all the methods needed to import +the nodes, users and slices from OAR and LDAP to the SFA database. +Also creates the iotlab specific databse and table to keep track +of which slice hrn contains which job. +""" +from sfa.util.config import Config +from sfa.util.xrn import Xrn, get_authority, hrn_to_urn + +from sfa.iotlab.iotlabdriver import IotlabDriver +from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB +from sfa.trust.certificate import Keypair, convert_public_key +from sfa.trust.gid import create_uuid + +from sfa.storage.alchemy import dbsession +from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, \ + RegUser, RegKey + + +from sqlalchemy.exc import SQLAlchemyError + + +class IotlabImporter: + """ + IotlabImporter class, generic importer_class. Used to populate the SFA DB + with iotlab resources' records. + Used to update records when new resources, users or nodes, are added + or deleted. + """ + + def __init__(self, auth_hierarchy, loc_logger): + """ + Sets and defines import logger and the authority name. Gathers all the + records already registerd in the SFA DB, broke them into 3 dicts, by + type and hrn, by email and by type and pointer. + + :param auth_hierarchy: authority name + :type auth_hierarchy: string + :param loc_logger: local logger + :type loc_logger: _SfaLogger + + """ + self.auth_hierarchy = auth_hierarchy + self.logger = loc_logger + self.logger.setLevelDebug() + #retrieve all existing SFA objects + self.all_records = dbsession.query(RegRecord).all() + + # initialize record.stale to True by default, + # then mark stale=False on the ones that are in use + for record in self.all_records: + record.stale = True + #create hash by (type,hrn) + #used to know if a given record is already known to SFA + self.records_by_type_hrn = \ + dict([((record.type, record.hrn), record) + for record in self.all_records]) + + self.users_rec_by_email = \ + dict([(record.email, record) + for record in self.all_records if record.type == 'user']) + + # create hash by (type,pointer) + self.records_by_type_pointer = \ + dict([((str(record.type), record.pointer), record) + for record in self.all_records if record.pointer != -1]) + + + @staticmethod + def hostname_to_hrn_escaped(root_auth, hostname): + """ + + Returns a node's hrn based on its hostname and the root authority and by + removing special caracters from the hostname. + + :param root_auth: root authority name + :param hostname: nodes's hostname + :type root_auth: string + :type hostname: string + :rtype: string + """ + return '.'.join([root_auth, Xrn.escape(hostname)]) + + + @staticmethod + def slicename_to_hrn(person_hrn): + """ + + Returns the slicename associated to a given person's hrn. + + :param person_hrn: user's hrn + :type person_hrn: string + :rtype: string + """ + return (person_hrn + '_slice') + + def add_options(self, parser): + """ + .. warning:: not used + """ + # we don't have any options for now + pass + + def find_record_by_type_hrn(self, record_type, hrn): + """ + Finds the record associated with the hrn and its type given in parameter + if the tuple (hrn, type hrn) is an existing key in the dictionary. + + :param record_type: the record's type (slice, node, authority...) + :type record_type: string + :param hrn: Human readable name of the object's record + :type hrn: string + :returns: Returns the record associated with a given hrn and hrn type. + Returns None if the key tuple is not in the dictionary. + :rtype: RegUser if user, RegSlice if slice, RegNode if node...or None if + record does not exist. + + """ + return self.records_by_type_hrn.get((record_type, hrn), None) + + def locate_by_type_pointer(self, record_type, pointer): + """ + + Returns the record corresponding to the key pointer and record + type. Returns None if the record does not exist and is not in the + records_by_type_pointer dictionnary. + + :param record_type: the record's type (slice, node, authority...) + :type record_type: string + :param pointer:Pointer to where the record is in the origin db, + used in case the record comes from a trusted authority. + :type pointer: integer + :rtype: RegUser if user, RegSlice if slice, RegNode if node... + or None if record does not exist. + """ + return self.records_by_type_pointer.get((record_type, pointer), None) + + + def update_just_added_records_dict(self, record): + """ + + Updates the records_by_type_hrn dictionnary if the record has + just been created. + + :param record: Record to add in the records_by_type_hrn dict. + :type record: dictionary + """ + rec_tuple = (record.type, record.hrn) + if rec_tuple in self.records_by_type_hrn: + self.logger.warning("IotlabImporter.update_just_added_records_dict:\ + duplicate (%s,%s)" % rec_tuple) + return + self.records_by_type_hrn[rec_tuple] = record + + + def import_nodes(self, site_node_ids, nodes_by_id, iotlabdriver): + """ + + Creates appropriate hostnames and RegNode records for each node in + site_node_ids, based on the information given by the dict nodes_by_id + that was made from data from OAR. Saves the records to the DB. + + :param site_node_ids: site's node ids + :type site_node_ids: list of integers + :param nodes_by_id: dictionary , key is the node id, value is the a dict + with node information. + :type nodes_by_id: dictionary + :param iotlabdriver: IotlabDriver object, used to have access to + iotlabdriver attributes. + :type iotlabdriver: IotlabDriver + + :returns: None + :rtype: None + + """ + + for node_id in site_node_ids: + try: + node = nodes_by_id[node_id] + except KeyError: + self.logger.warning("IotlabImporter: cannot find node_id %s \ + - ignored" % (node_id)) + continue + escaped_hrn = \ + self.hostname_to_hrn_escaped(iotlabdriver.iotlab_api.root_auth, + node['hostname']) + self.logger.info("IOTLABIMPORTER node %s " % (node)) + hrn = node['hrn'] + + # xxx this sounds suspicious + if len(hrn) > 64: + hrn = hrn[:64] + node_record = self.find_record_by_type_hrn('node', hrn) + if not node_record: + pkey = Keypair(create=True) + urn = hrn_to_urn(escaped_hrn, 'node') + node_gid = \ + self.auth_hierarchy.create_gid(urn, create_uuid(), pkey) + + def iotlab_get_authority(hrn): + """ Gets the authority part in the hrn. + :param hrn: hrn whose authority we are looking for. + :type hrn: string + :returns: splits the hrn using the '.' separator and returns + the authority part of the hrn. + :rtype: string + + """ + return hrn.split(".")[0] + + node_record = RegNode(hrn=hrn, gid=node_gid, + pointer='-1', + authority=iotlab_get_authority(hrn)) + try: + + node_record.just_created() + dbsession.add(node_record) + dbsession.commit() + self.logger.info("IotlabImporter: imported node: %s" + % node_record) + self.update_just_added_records_dict(node_record) + except SQLAlchemyError: + self.logger.log_exc("IotlabImporter: failed to import node") + else: + #TODO: xxx update the record ... + pass + node_record.stale = False + + def import_sites_and_nodes(self, iotlabdriver): + """ + + Gets all the sites and nodes from OAR, process the information, + creates hrns and RegAuthority for sites, and feed them to the database. + For each site, import the site's nodes to the DB by calling + import_nodes. + + :param iotlabdriver: IotlabDriver object, used to have access to + iotlabdriver methods and fetching info on sites and nodes. + :type iotlabdriver: IotlabDriver + """ + + sites_listdict = iotlabdriver.iotlab_api.GetSites() + nodes_listdict = iotlabdriver.iotlab_api.GetNodes() + nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict]) + for site in sites_listdict: + site_hrn = site['name'] + site_record = self.find_record_by_type_hrn ('authority', site_hrn) + self.logger.info("IotlabImporter: import_sites_and_nodes \ + (site) %s \r\n " % site_record) + if not site_record: + try: + urn = hrn_to_urn(site_hrn, 'authority') + if not self.auth_hierarchy.auth_exists(urn): + self.auth_hierarchy.create_auth(urn) + + auth_info = self.auth_hierarchy.get_auth_info(urn) + site_record = \ + RegAuthority(hrn=site_hrn, + gid=auth_info.get_gid_object(), + pointer='-1', + authority=get_authority(site_hrn)) + site_record.just_created() + dbsession.add(site_record) + dbsession.commit() + self.logger.info("IotlabImporter: imported authority \ + (site) %s" % site_record) + self.update_just_added_records_dict(site_record) + except SQLAlchemyError: + # if the site import fails then there is no point in + # trying to import the + # site's child records(node, slices, persons), so skip them. + self.logger.log_exc("IotlabImporter: failed to import \ + site. Skipping child records") + continue + else: + # xxx update the record ... + pass + + site_record.stale = False + self.import_nodes(site['node_ids'], nodes_by_id, iotlabdriver) + + return + + + + def init_person_key(self, person, iotlab_key): + """ + + Returns a tuple pubkey and pkey. + + :param person Person's data. + :type person: dict + :param iotlab_key: SSH public key, from LDAP user's data. + RSA type supported. + :type iotlab_key: string + :rtype (string, Keypair) + + """ + pubkey = None + if person['pkey']: + # randomly pick first key in set + pubkey = iotlab_key + + try: + pkey = convert_public_key(pubkey) + except TypeError: + #key not good. create another pkey + self.logger.warn("IotlabImporter: \ + unable to convert public \ + key for %s" % person['hrn']) + pkey = Keypair(create=True) + + else: + # the user has no keys. + #Creating a random keypair for the user's gid + self.logger.warn("IotlabImporter: person %s does not have a \ + public key" % (person['hrn'])) + pkey = Keypair(create=True) + return (pubkey, pkey) + + def import_persons_and_slices(self, iotlabdriver): + """ + + Gets user data from LDAP, process the information. + Creates hrn for the user's slice, the user's gid, creates + the RegUser record associated with user. Creates the RegKey record + associated nwith the user's key. + Saves those records into the SFA DB. + import the user's slice onto the database as well by calling + import_slice. + + :param iotlabdriver: IotlabDriver object, used to have access to + iotlabdriver attributes. + :type iotlabdriver: IotlabDriver + """ + ldap_person_listdict = iotlabdriver.iotlab_api.GetPersons() + self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n" + % (ldap_person_listdict)) + + # import persons + for person in ldap_person_listdict: + + self.logger.info("IotlabImporter: person :" % (person)) + if 'ssh-rsa' not in person['pkey']: + #people with invalid ssh key (ssh-dss, empty, bullshit keys...) + #won't be imported + continue + person_hrn = person['hrn'] + slice_hrn = self.slicename_to_hrn(person['hrn']) + + # xxx suspicious again + if len(person_hrn) > 64: + person_hrn = person_hrn[:64] + person_urn = hrn_to_urn(person_hrn, 'user') + + + self.logger.info("IotlabImporter: users_rec_by_email %s " + % (self.users_rec_by_email)) + + #Check if user using person['email'] from LDAP is already registered + #in SFA. One email = one person. In this case, do not create another + #record for this person + #person_hrn returned by GetPerson based on iotlab root auth + + #uid ldap + user_record = self.find_record_by_type_hrn('user', person_hrn) + + if not user_record and person['email'] in self.users_rec_by_email: + user_record = self.users_rec_by_email[person['email']] + person_hrn = user_record.hrn + person_urn = hrn_to_urn(person_hrn, 'user') + + + slice_record = self.find_record_by_type_hrn('slice', slice_hrn) + + iotlab_key = person['pkey'] + # new person + if not user_record: + (pubkey, pkey) = self.init_person_key(person, iotlab_key) + if pubkey is not None and pkey is not None: + person_gid = \ + self.auth_hierarchy.create_gid(person_urn, + create_uuid(), pkey) + if person['email']: + self.logger.debug("IOTLAB IMPORTER \ + PERSON EMAIL OK email %s " % (person['email'])) + person_gid.set_email(person['email']) + user_record = \ + RegUser(hrn=person_hrn, + gid=person_gid, + pointer='-1', + authority=get_authority(person_hrn), + email=person['email']) + else: + user_record = \ + RegUser(hrn=person_hrn, + gid=person_gid, + pointer='-1', + authority=get_authority(person_hrn)) + + if pubkey: + user_record.reg_keys = [RegKey(pubkey)] + else: + self.logger.warning("No key found for user %s" + % (user_record)) + + try: + user_record.just_created() + dbsession.add (user_record) + dbsession.commit() + self.logger.info("IotlabImporter: imported person \ + %s" % (user_record)) + self.update_just_added_records_dict(user_record) + + except SQLAlchemyError: + self.logger.log_exc("IotlabImporter: \ + failed to import person %s" % (person)) + else: + # update the record ? + # if user's primary key has changed then we need to update + # the users gid by forcing an update here + sfa_keys = user_record.reg_keys + + new_key = False + if iotlab_key is not sfa_keys: + new_key = True + if new_key: + self.logger.info("IotlabImporter: \t \t USER UPDATE \ + person: %s" % (person['hrn'])) + (pubkey, pkey) = self.init_person_key(person, iotlab_key) + person_gid = \ + self.auth_hierarchy.create_gid(person_urn, + create_uuid(), pkey) + if not pubkey: + user_record.reg_keys = [] + else: + user_record.reg_keys = [RegKey(pubkey)] + self.logger.info("IotlabImporter: updated person: %s" + % (user_record)) + + if person['email']: + user_record.email = person['email'] + + try: + dbsession.commit() + user_record.stale = False + except SQLAlchemyError: + self.logger.log_exc("IotlabImporter: \ + failed to update person %s"% (person)) + + self.import_slice(slice_hrn, slice_record, user_record) + + + def import_slice(self, slice_hrn, slice_record, user_record): + """ + + Create RegSlice record according to the slice hrn if the slice + does not exist yet.Creates a relationship with the user record + associated with the slice. + Commit the record to the database. + + + :param slice_hrn: Human readable name of the slice. + :type slice_hrn: string + :param slice_record: record of the slice found in the DB, if any. + :type slice_record: RegSlice or None + :param user_record: user record found in the DB if any. + :type user_record: RegUser + + .. todo::Update the record if a slice record already exists. + """ + if not slice_record: + pkey = Keypair(create=True) + urn = hrn_to_urn(slice_hrn, 'slice') + slice_gid = \ + self.auth_hierarchy.create_gid(urn, + create_uuid(), pkey) + slice_record = RegSlice(hrn=slice_hrn, gid=slice_gid, + pointer='-1', + authority=get_authority(slice_hrn)) + try: + slice_record.just_created() + dbsession.add(slice_record) + dbsession.commit() + + + self.update_just_added_records_dict(slice_record) + + except SQLAlchemyError: + self.logger.log_exc("IotlabImporter: failed to import slice") + + #No slice update upon import in iotlab + else: + # xxx update the record ... + self.logger.warning("Slice update not yet implemented") + pass + # record current users affiliated with the slice + + + slice_record.reg_researchers = [user_record] + try: + dbsession.commit() + slice_record.stale = False + except SQLAlchemyError: + self.logger.log_exc("IotlabImporter: failed to update slice") + + + def run(self, options): + """ + Create the special iotlab table, testbed_xp, in the iotlab database. + Import everything (users, slices, nodes and sites from OAR + and LDAP) into the SFA database. + Delete stale records that are no longer in OAR or LDAP. + :param options: + :type options: + """ + config = Config() + + iotlabdriver = IotlabDriver(config) + iotlab_db = TestbedAdditionalSfaDB(config) + #Create special slice table for iotlab + + if not iotlab_db.exists('testbed_xp'): + iotlab_db.createtable() + self.logger.info("IotlabImporter.run: testbed_xp table created ") + + # import site and node records in site into the SFA db. + self.import_sites_and_nodes(iotlabdriver) + #import users and slice into the SFA DB. + self.import_persons_and_slices(iotlabdriver) + + ### remove stale records + # special records must be preserved + system_hrns = [iotlabdriver.hrn, iotlabdriver.iotlab_api.root_auth, + iotlabdriver.hrn + '.slicemanager'] + for record in self.all_records: + if record.hrn in system_hrns: + record.stale = False + if record.peer_authority: + record.stale = False + + for record in self.all_records: + if record.type == 'user': + self.logger.info("IotlabImporter: stale records: hrn %s %s" + % (record.hrn, record.stale)) + try: + stale = record.stale + except: + stale = True + self.logger.warning("stale not found with %s" % record) + if stale: + self.logger.info("IotlabImporter: deleting stale record: %s" + % (record)) + + try: + dbsession.delete(record) + dbsession.commit() + except SQLAlchemyError: + self.logger.log_exc("IotlabImporter: failed to delete \ + stale record %s" % (record)) diff --git a/sfa/importer/plimporter.py b/sfa/importer/plimporter.py index ff3afc49..41325a89 100644 --- a/sfa/importer/plimporter.py +++ b/sfa/importer/plimporter.py @@ -145,7 +145,7 @@ class PlImporter: # Get all plc sites # retrieve only required stuf sites = shell.GetSites({'peer_id': None, 'enabled' : True}, - ['site_id','login_base','node_ids','slice_ids','person_ids',]) + ['site_id','login_base','node_ids','slice_ids','person_ids', 'name']) # create a hash of sites by login_base # sites_by_login_base = dict ( [ ( site['login_base'], site ) for site in sites ] ) # Get all plc users @@ -192,6 +192,9 @@ class PlImporter: # start importing for site in sites: + if site['name'].startswith('sfa.'): + continue + site_hrn = _get_site_hrn(interface_hrn, site) # import if hrn is not in list of existing hrns or if the hrn exists # but its not a site record @@ -254,7 +257,7 @@ class PlImporter: pass node_record.stale=False - site_pis=set() + site_pis=[] # import persons for person_id in site['person_ids']: proceed=False @@ -362,7 +365,7 @@ class PlImporter: # this is valid for all sites she is in.. # PI is coded with role_id==20 if 20 in person['role_ids']: - site_pis.add (user_record) + site_pis.append (user_record) except: self.logger.log_exc("PlImporter: failed to import person %d %s"%(person['person_id'],person['email'])) @@ -373,7 +376,7 @@ class PlImporter: # being improperly handled, and where the whole loop on persons # could be performed twice with the same person... # so hopefully we do not need to eliminate duplicates explicitly here anymore - site_record.reg_pis = list(site_pis) + site_record.reg_pis = list(set(site_pis)) dbsession.commit() # import slices @@ -400,15 +403,10 @@ class PlImporter: except: self.logger.log_exc("PlImporter: failed to import slice %s (%s)"%(slice_hrn,slice['name'])) else: - # update the pointer if it has changed - if slice_id != slice_record.pointer: - self.logger.info("updating record (slice) pointer") - slice_record.pointer = slice_id - dbsession.commit() # xxx update the record ... # given that we record the current set of users anyways, there does not seem to be much left to do here # self.logger.warning ("Slice update not yet implemented on slice %s (%s)"%(slice_hrn,slice['name'])) - #pass + pass # record current users affiliated with the slice slice_record.reg_researchers = \ [ self.locate_by_type_pointer ('user',user_id) for user_id in slice['person_ids'] ] diff --git a/sfa/importer/slabimporter.py b/sfa/importer/slabimporter.py deleted file mode 100644 index 60243a29..00000000 --- a/sfa/importer/slabimporter.py +++ /dev/null @@ -1,542 +0,0 @@ -from sfa.util.config import Config -from sfa.util.xrn import Xrn, get_authority, hrn_to_urn - -from sfa.senslab.slabdriver import SlabDriver - -from sfa.trust.certificate import Keypair, convert_public_key -from sfa.trust.gid import create_uuid - -from sfa.storage.alchemy import dbsession -from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, \ - RegUser, RegKey - - -from sqlalchemy.exc import SQLAlchemyError - - - -class SlabImporter: - """ - SlabImporter class, generic importer_class. Used to populate the SFA DB - with senslab resources' records. - Used to update records when new resources, users or nodes, are added - or deleted. - """ - - def __init__ (self, auth_hierarchy, loc_logger): - """ - Sets and defines import logger and the authority name. Gathers all the - records already registerd in the SFA DB, broke them into 3 dicts, - by type and hrn, by email and by type and pointer. - - :param auth_hierarchy: authority name - :type auth_hierarchy: string - :param loc_logger: local logger - :type loc_logger: _SfaLogger - - """ - self.auth_hierarchy = auth_hierarchy - self.logger = loc_logger - self.logger.setLevelDebug() - #retrieve all existing SFA objects - self.all_records = dbsession.query(RegRecord).all() - - # initialize record.stale to True by default, - # then mark stale=False on the ones that are in use - for record in self.all_records: - record.stale = True - #create hash by (type,hrn) - #used to know if a given record is already known to SFA - self.records_by_type_hrn = \ - dict([( (record.type,record.hrn), record) \ - for record in self.all_records]) - - self.users_rec_by_email = \ - dict([ (record.email, record) \ - for record in self.all_records if record.type == 'user']) - - # create hash by (type,pointer) - self.records_by_type_pointer = \ - dict([ ( (str(record.type), record.pointer) , record) \ - for record in self.all_records if record.pointer != -1]) - - - - @staticmethod - def hostname_to_hrn_escaped(root_auth, hostname): - """ - - Returns a node's hrn based on its hostname and the root - authority and by removing special caracters from the hostname. - - :param root_auth: root authority name - :param hostname: nodes's hostname - :type root_auth: string - :type hostname: string - :rtype: string - """ - return '.'.join( [root_auth, Xrn.escape(hostname)] ) - - - @staticmethod - def slicename_to_hrn(person_hrn): - """ - - Returns the slicename associated to a given person's hrn. - - :param person_hrn: user's hrn - :type person_hrn: string - :rtype: string - """ - return (person_hrn +'_slice') - - def add_options (self, parser): - # we don't have any options for now - pass - - def find_record_by_type_hrn(self, record_type, hrn): - """ - - Returns the record associated with a given hrn and hrn type. - Returns None if the key tuple is not in dictionary. - - :param record_type: the record's type (slice, node, authority...) - :type record_type: string - :param hrn: Human readable name of the object's record - :type hrn: string - :rtype: RegUser if user, RegSlice if slice, RegNode if node... - or None if record does not exist. - - """ - return self.records_by_type_hrn.get ( (record_type, hrn), None) - - def locate_by_type_pointer (self, record_type, pointer): - """ - - Returns the record corresponding to the key pointer and record - type. Returns None if the record does not exist and is not in the - records_by_type_pointer dictionnary. - - :param record_type: the record's type (slice, node, authority...) - :type record_type: string - :param pointer:Pointer to where the record is in the origin db, - used in case the record comes from a trusted authority. - :type pointer: integer - :rtype: RegUser if user, RegSlice if slice, RegNode if node... - or None if record does not exist. - """ - return self.records_by_type_pointer.get ( (record_type, pointer), None) - - - def update_just_added_records_dict (self, record): - """ - - Updates the records_by_type_hrn dictionnary if the record has - just been created. - - :param record: Record to add in the records_by_type_hrn dict. - :type record: dictionary - """ - rec_tuple = (record.type, record.hrn) - if rec_tuple in self.records_by_type_hrn: - self.logger.warning ("SlabImporter.update_just_added_records_dict:\ - duplicate (%s,%s)"%rec_tuple) - return - self.records_by_type_hrn [ rec_tuple ] = record - - def import_sites_and_nodes(self, slabdriver): - """ - - Gets all the sites and nodes from OAR, process the information, - creates hrns and RegAuthority for sites, and feed them to the database. - For each site, import the site's nodes to the DB by calling - import_nodes. - - :param slabdriver: SlabDriver object, used to have access to slabdriver - methods and fetching info on sites and nodes. - :type slabdriver: SlabDriver - """ - - sites_listdict = slabdriver.slab_api.GetSites() - nodes_listdict = slabdriver.slab_api.GetNodes() - nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict]) - for site in sites_listdict: - site_hrn = site['name'] - site_record = self.find_record_by_type_hrn ('authority', site_hrn) - if not site_record: - try: - urn = hrn_to_urn(site_hrn, 'authority') - if not self.auth_hierarchy.auth_exists(urn): - self.auth_hierarchy.create_auth(urn) - - auth_info = self.auth_hierarchy.get_auth_info(urn) - site_record = RegAuthority(hrn=site_hrn, \ - gid=auth_info.get_gid_object(), - pointer='-1', - authority=get_authority(site_hrn)) - site_record.just_created() - dbsession.add(site_record) - dbsession.commit() - self.logger.info("SlabImporter: imported authority (site) \ - %s" % site_record) - self.update_just_added_records_dict(site_record) - except SQLAlchemyError: - # if the site import fails then there is no point in - # trying to import the - # site's child records(node, slices, persons), so skip them. - self.logger.log_exc("SlabImporter: failed to import site. \ - Skipping child records") - continue - else: - # xxx update the record ... - pass - - - site_record.stale = False - self.import_nodes(site['node_ids'], nodes_by_id, slabdriver) - - return - - def import_nodes(self, site_node_ids, nodes_by_id, slabdriver) : - """ - - Creates appropriate hostnames and RegNode records for - each node in site_node_ids, based on the information given by the - dict nodes_by_id that was made from data from OAR. - Saves the records to the DB. - - :param site_node_ids: site's node ids - :type site_node_ids: list of integers - :param nodes_by_id: dictionary , key is the node id, value is the a dict - with node information. - :type nodes_by_id: dictionary - :param slabdriver:SlabDriver object, used to have access to slabdriver - attributes. - :type slabdriver:SlabDriver - - """ - - for node_id in site_node_ids: - try: - node = nodes_by_id[node_id] - except KeyError: - self.logger.warning ("SlabImporter: cannot find node_id %s \ - - ignored" %(node_id)) - continue - escaped_hrn = \ - self.hostname_to_hrn_escaped(slabdriver.slab_api.root_auth, \ - node['hostname']) - self.logger.info("SLABIMPORTER node %s " %(node)) - hrn = node['hrn'] - - - # xxx this sounds suspicious - if len(hrn) > 64: - hrn = hrn[:64] - node_record = self.find_record_by_type_hrn( 'node', hrn ) - if not node_record: - pkey = Keypair(create=True) - urn = hrn_to_urn(escaped_hrn, 'node') - node_gid = \ - self.auth_hierarchy.create_gid(urn, \ - create_uuid(), pkey) - - def slab_get_authority(hrn): - return hrn.split(".")[0] - - node_record = RegNode(hrn=hrn, gid=node_gid, - pointer = '-1', - authority=slab_get_authority(hrn)) - try: - - node_record.just_created() - dbsession.add(node_record) - dbsession.commit() - self.logger.info("SlabImporter: imported node: %s" \ - % node_record) - self.update_just_added_records_dict(node_record) - except SQLAlchemyError: - self.logger.log_exc("SlabImporter: \ - failed to import node") - else: - #TODO: xxx update the record ... - pass - node_record.stale = False - - - def init_person_key (self, person, slab_key): - """ - - Returns a tuple pubkey and pkey. - - :param person Person's data. - :type person: dict - :param slab_key: SSH public key, from LDAP user's data. - RSA type supported. - :type slab_key: string - :rtype (string, Keypair) - """ - pubkey = None - if person['pkey']: - # randomly pick first key in set - pubkey = slab_key - - try: - pkey = convert_public_key(pubkey) - except TypeError: - #key not good. create another pkey - self.logger.warn('SlabImporter: \ - unable to convert public \ - key for %s' %person['hrn']) - pkey = Keypair(create=True) - - else: - # the user has no keys. - #Creating a random keypair for the user's gid - self.logger.warn("SlabImporter: person %s does not have a \ - public key" %(person['hrn'])) - pkey = Keypair(create=True) - return (pubkey, pkey) - - - def import_persons_and_slices(self, slabdriver): - """ - - Gets user data from LDAP, process the information. - Creates hrn for the user's slice, the user's gid, creates - the RegUser record associated with user. Creates the RegKey record - associated nwith the user's key. - Saves those records into the SFA DB. - import the user's slice onto the database as well by calling - import_slice. - - :param slabdriver:SlabDriver object, used to have access to slabdriver - attributes. - :type slabdriver:SlabDriver - """ - ldap_person_listdict = slabdriver.slab_api.GetPersons() - self.logger.info("SLABIMPORT \t ldap_person_listdict %s \r\n" \ - %(ldap_person_listdict)) - - # import persons - for person in ldap_person_listdict : - - self.logger.info("SlabImporter: person :" %(person)) - if 'ssh-rsa' not in person['pkey']: - #people with invalid ssh key (ssh-dss, empty, bullshit keys...) - #won't be imported - continue - person_hrn = person['hrn'] - slice_hrn = self.slicename_to_hrn(person['hrn']) - - # xxx suspicious again - if len(person_hrn) > 64: - person_hrn = person_hrn[:64] - person_urn = hrn_to_urn(person_hrn, 'user') - - - self.logger.info("SlabImporter: users_rec_by_email %s " \ - %(self.users_rec_by_email)) - - #Check if user using person['email'] from LDAP is already registered - #in SFA. One email = one person. In this case, do not create another - #record for this person - #person_hrn returned by GetPerson based on senslab root auth + - #uid ldap - user_record = self.find_record_by_type_hrn('user', person_hrn) - - if not user_record and person['email'] in self.users_rec_by_email: - user_record = self.users_rec_by_email[person['email']] - person_hrn = user_record.hrn - person_urn = hrn_to_urn(person_hrn, 'user') - - - slice_record = self.find_record_by_type_hrn ('slice', slice_hrn) - - slab_key = person['pkey'] - # new person - if not user_record: - (pubkey, pkey) = self.init_person_key(person, slab_key) - if pubkey is not None and pkey is not None : - person_gid = \ - self.auth_hierarchy.create_gid(person_urn, \ - create_uuid(), pkey) - if person['email']: - self.logger.debug( "SLAB IMPORTER \ - PERSON EMAIL OK email %s " %(person['email'])) - person_gid.set_email(person['email']) - user_record = RegUser(hrn=person_hrn, \ - gid=person_gid, - pointer='-1', - authority=get_authority(person_hrn), - email=person['email']) - else: - user_record = RegUser(hrn=person_hrn, \ - gid=person_gid, - pointer='-1', - authority=get_authority(person_hrn)) - - if pubkey: - user_record.reg_keys = [RegKey(pubkey)] - else: - self.logger.warning("No key found for user %s" \ - %(user_record)) - - try: - user_record.just_created() - dbsession.add (user_record) - dbsession.commit() - self.logger.info("SlabImporter: imported person %s"\ - %(user_record)) - self.update_just_added_records_dict( user_record ) - - except SQLAlchemyError: - self.logger.log_exc("SlabImporter: \ - failed to import person %s"%(person)) - else: - # update the record ? - # if user's primary key has changed then we need to update - # the users gid by forcing an update here - sfa_keys = user_record.reg_keys - - new_key = False - if slab_key is not sfa_keys : - new_key = True - if new_key: - self.logger.info("SlabImporter: \t \t USER UPDATE \ - person: %s" %(person['hrn'])) - (pubkey, pkey) = self.init_person_key (person, slab_key) - person_gid = \ - self.auth_hierarchy.create_gid(person_urn, \ - create_uuid(), pkey) - if not pubkey: - user_record.reg_keys = [] - else: - user_record.reg_keys = [RegKey(pubkey)] - self.logger.info("SlabImporter: updated person: %s" \ - % (user_record)) - - if person['email']: - user_record.email = person['email'] - - try: - dbsession.commit() - user_record.stale = False - except SQLAlchemyError: - self.logger.log_exc("SlabImporter: \ - failed to update person %s"%(person)) - - self.import_slice(slice_hrn, slice_record, user_record) - - - def import_slice(self, slice_hrn, slice_record, user_record): - """ - - Create RegSlice record according to the slice hrn if the slice - does not exist yet.Creates a relationship with the user record - associated with the slice. - Commit the record to the database. - - - :param slice_hrn: Human readable name of the slice. - :type slice_hrn: string - :param slice_record: record of the slice found in the DB, if any. - :type slice_record: RegSlice or None - :param user_record: user record found in the DB if any. - :type user_record: RegUser - - .. todo::Update the record if a slice record already exists. - """ - if not slice_record : - pkey = Keypair(create=True) - urn = hrn_to_urn(slice_hrn, 'slice') - slice_gid = \ - self.auth_hierarchy.create_gid(urn, \ - create_uuid(), pkey) - slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid, - pointer='-1', - authority=get_authority(slice_hrn)) - try: - slice_record.just_created() - dbsession.add(slice_record) - dbsession.commit() - - - self.update_just_added_records_dict ( slice_record ) - - except SQLAlchemyError: - self.logger.log_exc("SlabImporter: failed to import slice") - - #No slice update upon import in senslab - else: - # xxx update the record ... - self.logger.warning ("Slice update not yet implemented") - pass - # record current users affiliated with the slice - - - slice_record.reg_researchers = [user_record] - try: - dbsession.commit() - slice_record.stale = False - except SQLAlchemyError: - self.logger.log_exc("SlabImporter: failed to update slice") - - - def run (self, options): - """ - Create the special senslab table, slab_xp, in the senslab database. - Import everything (users, slices, nodes and sites from OAR - and LDAP) into the SFA database. - Delete stale records that are no longer in OAR or LDAP. - :param options: - :type options: - """ - config = Config() - - slabdriver = SlabDriver(config) - - #Create special slice table for senslab - - if not slabdriver.db.exists('slab_xp'): - slabdriver.db.createtable() - self.logger.info ("SlabImporter.run: slab_xp table created ") - - - # import site and node records in site into the SFA db. - self.import_sites_and_nodes(slabdriver) - #import users and slice into the SFA DB. - self.import_persons_and_slices(slabdriver) - - ### remove stale records - # special records must be preserved - system_hrns = [slabdriver.hrn, slabdriver.slab_api.root_auth, \ - slabdriver.hrn+ '.slicemanager'] - for record in self.all_records: - if record.hrn in system_hrns: - record.stale = False - if record.peer_authority: - record.stale = False - - - for record in self.all_records: - if record.type == 'user': - self.logger.info("SlabImporter: stale records: hrn %s %s" \ - %(record.hrn,record.stale) ) - try: - stale = record.stale - except : - stale = True - self.logger.warning("stale not found with %s"%record) - if stale: - self.logger.info("SlabImporter: deleting stale record: %s" \ - %(record)) - - try: - dbsession.delete(record) - dbsession.commit() - except SQLAlchemyError: - self.logger.log_exc("SlabImporter: failed to delete stale \ - record %s" %(record) ) - - diff --git a/sfa/iotlab/LDAPapi.py b/sfa/iotlab/LDAPapi.py new file mode 100644 index 00000000..15067ac5 --- /dev/null +++ b/sfa/iotlab/LDAPapi.py @@ -0,0 +1,1024 @@ +""" +This API is adapted for OpenLDAP. The file contains all LDAP classes and methods +needed to: +- Load the LDAP connection configuration file (login, address..) with LdapConfig +- Connect to LDAP with ldap_co +- Create a unique LDAP login and password for a user based on his email or last +name and first name with LoginPassword. +- Manage entries in LDAP using SFA records with LDAPapi (Search, Add, Delete, +Modify) + +""" +import random +from passlib.hash import ldap_salted_sha1 as lssha + +from sfa.util.xrn import get_authority +from sfa.util.sfalogging import logger +from sfa.util.config import Config + +import ldap +import ldap.modlist as modlist + +import os.path + + +class LdapConfig(): + """ + Ldap configuration class loads the configuration file and sets the + ldap IP address, password, people dn, web dn, group dn. All these settings + were defined in a separate file ldap_config.py to avoid sharing them in + the SFA git as it contains sensible information. + + """ + def __init__(self, config_file='/etc/sfa/ldap_config.py'): + """Loads configuration from file /etc/sfa/ldap_config.py and set the + parameters for connection to LDAP. + + """ + + try: + execfile(config_file, self.__dict__) + + self.config_file = config_file + # path to configuration data + self.config_path = os.path.dirname(config_file) + except IOError: + raise IOError, "Could not find or load the configuration file: %s" \ + % config_file + + +class ldap_co: + """ Set admin login and server configuration variables.""" + + def __init__(self): + """Fetch LdapConfig attributes (Ldap server connection parameters and + defines port , version and subtree scope. + + """ + #Iotlab PROD LDAP parameters + self.ldapserv = None + ldap_config = LdapConfig() + self.config = ldap_config + self.ldapHost = ldap_config.LDAP_IP_ADDRESS + self.ldapPeopleDN = ldap_config.LDAP_PEOPLE_DN + self.ldapGroupDN = ldap_config.LDAP_GROUP_DN + self.ldapAdminDN = ldap_config.LDAP_WEB_DN + self.ldapAdminPassword = ldap_config.LDAP_WEB_PASSWORD + self.ldapPort = ldap.PORT + self.ldapVersion = ldap.VERSION3 + self.ldapSearchScope = ldap.SCOPE_SUBTREE + + def connect(self, bind=True): + """Enables connection to the LDAP server. + + :param bind: Set the bind parameter to True if a bind is needed + (for add/modify/delete operations). Set to False otherwise. + :type bind: boolean + :returns: dictionary with status of the connection. True if Successful, + False if not and in this case the error + message( {'bool', 'message'} ). + :rtype: dict + + """ + try: + self.ldapserv = ldap.open(self.ldapHost) + except ldap.LDAPError, error: + return {'bool': False, 'message': error} + + # Bind with authentification + if(bind): + return self.bind() + + else: + return {'bool': True} + + def bind(self): + """ Binding method. + + :returns: dictionary with the bind status. True if Successful, + False if not and in this case the error message({'bool','message'}) + :rtype: dict + + """ + try: + # Opens a connection after a call to ldap.open in connect: + self.ldapserv = ldap.initialize("ldap://" + self.ldapHost) + + # Bind/authenticate with a user with apropriate + #rights to add objects + self.ldapserv.simple_bind_s(self.ldapAdminDN, + self.ldapAdminPassword) + + except ldap.LDAPError, error: + return {'bool': False, 'message': error} + + return {'bool': True} + + def close(self): + """Close the LDAP connection. + + Can throw an exception if the unbinding fails. + + :returns: dictionary with the bind status if the unbinding failed and + in this case the dict contains an error message. The dictionary keys + are : ({'bool','message'}) + :rtype: dict or None + + """ + try: + self.ldapserv.unbind_s() + except ldap.LDAPError, error: + return {'bool': False, 'message': error} + + +class LoginPassword(): + """ + + Class to handle login and password generation, using custom login generation + algorithm. + + """ + def __init__(self): + """ + + Sets password and login maximum length, and defines the characters that + can be found in a random generated password. + + """ + self.login_max_length = 8 + self.length_password = 8 + self.chars_password = ['!', '$', '(',')', '*', '+', ',', '-', '.', + '0', '1', '2', '3', '4', '5', '6', '7', '8', + '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', + 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + '_', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', + 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', + 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '\''] + + @staticmethod + def clean_user_names(record): + """ + + Removes special characters such as '-', '_' , '[', ']' and ' ' from the + first name and last name. + + :param record: user's record + :type record: dict + :returns: lower_first_name and lower_last_name if they were found + in the user's record. Return None, none otherwise. + :rtype: string, string or None, None. + + """ + if 'first_name' in record and 'last_name' in record: + #Remove all special characters from first_name/last name + lower_first_name = record['first_name'].replace('-', '')\ + .replace('_', '').replace('[', '')\ + .replace(']', '').replace(' ', '')\ + .lower() + lower_last_name = record['last_name'].replace('-', '')\ + .replace('_', '').replace('[', '')\ + .replace(']', '').replace(' ', '')\ + .lower() + return lower_first_name, lower_last_name + else: + return None, None + + @staticmethod + def extract_name_from_email(record): + """ + + When there is no valid first name and last name in the record, + the email is used to generate the login. Here, we assume the email + is firstname.lastname@something.smthg. The first name and last names + are extracted from the email, special charcaters are removed and + they are changed into lower case. + + :param record: user's data + :type record: dict + :returns: the first name and last name taken from the user's email. + lower_first_name, lower_last_name. + :rtype: string, string + + """ + + email = record['email'] + email = email.split('@')[0].lower() + lower_first_name = None + lower_last_name = None + #Assume there is first name and last name in email + #if there is a separator + separator_list = ['.', '_', '-'] + for sep in separator_list: + if sep in email: + mail = email.split(sep) + lower_first_name = mail[0] + lower_last_name = mail[1] + break + + #Otherwise just take the part before the @ as the + #lower_first_name and lower_last_name + if lower_first_name is None: + lower_first_name = email + lower_last_name = email + + return lower_first_name, lower_last_name + + def get_user_firstname_lastname(self, record): + """ + + Get the user first name and last name from the information we have in + the record. + + :param record: user's information + :type record: dict + :returns: the user's first name and last name. + + .. seealso:: clean_user_names + .. seealso:: extract_name_from_email + + """ + lower_first_name, lower_last_name = self.clean_user_names(record) + + #No first name and last name check email + if lower_first_name is None and lower_last_name is None: + + lower_first_name, lower_last_name = \ + self.extract_name_from_email(record) + + return lower_first_name, lower_last_name + + def choose_sets_chars_for_login(self, lower_first_name, lower_last_name): + """ + + Algorithm to select sets of characters from the first name and last + name, depending on the lenght of the last name and the maximum login + length which in our case is set to 8 characters. + + :param lower_first_name: user's first name in lower case. + :param lower_last_name: usr's last name in lower case. + :returns: user's login + :rtype: string + + """ + length_last_name = len(lower_last_name) + self.login_max_length = 8 + + #Try generating a unique login based on first name and last name + + if length_last_name >= self.login_max_length: + login = lower_last_name[0:self.login_max_length] + index = 0 + logger.debug("login : %s index : %s" % (login, index)) + elif length_last_name >= 4: + login = lower_last_name + index = 0 + logger.debug("login : %s index : %s" % (login, index)) + elif length_last_name == 3: + login = lower_first_name[0:1] + lower_last_name + index = 1 + logger.debug("login : %s index : %s" % (login, index)) + elif length_last_name == 2: + if len(lower_first_name) >= 2: + login = lower_first_name[0:2] + lower_last_name + index = 2 + logger.debug("login : %s index : %s" % (login, index)) + else: + logger.error("LoginException : \ + Generation login error with \ + minimum four characters") + + else: + logger.error("LDAP LdapGenerateUniqueLogin failed : \ + impossible to generate unique login for %s %s" + % (lower_first_name, lower_last_name)) + return index, login + + def generate_password(self): + """ + + Generate a password upon adding a new user in LDAP Directory + (8 characters length). The generated password is composed of characters + from the chars_password list. + + :returns: the randomly generated password + :rtype: string + + """ + password = str() + + length = len(self.chars_password) + for index in range(self.length_password): + char_index = random.randint(0, length - 1) + password += self.chars_password[char_index] + + return password + + @staticmethod + def encrypt_password(password): + """ + + Use passlib library to make a RFC2307 LDAP encrypted password salt size + is 8, use sha-1 algorithm. + + :param password: password not encrypted. + :type password: string + :returns: Returns encrypted password. + :rtype: string + + """ + #Keep consistency with Java Iotlab's LDAP API + #RFC2307SSHAPasswordEncryptor so set the salt size to 8 bytes + return lssha.encrypt(password, salt_size=8) + + +class LDAPapi: + """Defines functions to insert and search entries in the LDAP. + + .. note:: class supposes the unix schema is used + + """ + def __init__(self): + logger.setLevelDebug() + + #SFA related config + + config = Config() + self.login_pwd = LoginPassword() + self.authname = config.SFA_REGISTRY_ROOT_AUTH + self.conn = ldap_co() + self.ldapUserQuotaNFS = self.conn.config.LDAP_USER_QUOTA_NFS + self.ldapUserUidNumberMin = self.conn.config.LDAP_USER_UID_NUMBER_MIN + self.ldapUserGidNumber = self.conn.config.LDAP_USER_GID_NUMBER + self.ldapUserHomePath = self.conn.config.LDAP_USER_HOME_PATH + self.baseDN = self.conn.ldapPeopleDN + self.ldapShell = '/bin/bash' + + + def LdapGenerateUniqueLogin(self, record): + """ + + Generate login for adding a new user in LDAP Directory + (four characters minimum length). Get proper last name and + first name so that the user's login can be generated. + + :param record: Record must contain first_name and last_name. + :type record: dict + :returns: the generated login for the user described with record if the + login generation is successful, None if it fails. + :rtype: string or None + + """ + #For compatibility with other ldap func + if 'mail' in record and 'email' not in record: + record['email'] = record['mail'] + + lower_first_name, lower_last_name = \ + self.login_pwd.get_user_firstname_lastname(record) + + index, login = self.login_pwd.choose_sets_chars_for_login( + lower_first_name, lower_last_name) + + login_filter = '(uid=' + login + ')' + get_attrs = ['uid'] + try: + #Check if login already in use + + while (len(self.LdapSearch(login_filter, get_attrs)) is not 0): + + index += 1 + if index >= 9: + logger.error("LoginException : Generation login error \ + with minimum four characters") + else: + try: + login = \ + lower_first_name[0:index] + \ + lower_last_name[0: + self.login_pwd.login_max_length + - index] + login_filter = '(uid=' + login + ')' + except KeyError: + print "lower_first_name - lower_last_name too short" + + logger.debug("LDAP.API \t LdapGenerateUniqueLogin login %s" + % (login)) + return login + + except ldap.LDAPError, error: + logger.log_exc("LDAP LdapGenerateUniqueLogin Error %s" % (error)) + return None + + def find_max_uidNumber(self): + """Find the LDAP max uidNumber (POSIX uid attribute). + + Used when adding a new user in LDAP Directory + + :returns: max uidNumber + 1 + :rtype: string + + """ + #First, get all the users in the LDAP + get_attrs = "(uidNumber=*)" + login_filter = ['uidNumber'] + + result_data = self.LdapSearch(get_attrs, login_filter) + #It there is no user in LDAP yet, First LDAP user + if result_data == []: + max_uidnumber = self.ldapUserUidNumberMin + #Otherwise, get the highest uidNumber + else: + uidNumberList = [int(r[1]['uidNumber'][0])for r in result_data] + logger.debug("LDAPapi.py \tfind_max_uidNumber \ + uidNumberList %s " % (uidNumberList)) + max_uidnumber = max(uidNumberList) + 1 + + return str(max_uidnumber) + + + def get_ssh_pkey(self, record): + """TODO ; Get ssh public key from sfa record + To be filled by N. Turro ? or using GID pl way? + + """ + return 'A REMPLIR ' + + @staticmethod + #TODO Handle OR filtering in the ldap query when + #dealing with a list of records instead of doing a for loop in GetPersons + def make_ldap_filters_from_record(record=None): + """Helper function to make LDAP filter requests out of SFA records. + + :param record: user's sfa record. Should contain first_name,last_name, + email or mail, and if the record is enabled or not. If the dict + record does not have all of these, must at least contain the user's + email. + :type record: dict + :returns: LDAP request + :rtype: string + + """ + req_ldap = '' + req_ldapdict = {} + if record : + if 'first_name' in record and 'last_name' in record: + if record['first_name'] != record['last_name']: + req_ldapdict['cn'] = str(record['first_name'])+" "\ + + str(record['last_name']) + if 'email' in record: + req_ldapdict['mail'] = record['email'] + if 'mail' in record: + req_ldapdict['mail'] = record['mail'] + if 'enabled' in record: + if record['enabled'] is True: + req_ldapdict['shadowExpire'] = '-1' + else: + req_ldapdict['shadowExpire'] = '0' + + #Hrn should not be part of the filter because the hrn + #presented by a certificate of a SFA user not imported in + #Iotlab does not include the iotlab login in it + #Plus, the SFA user may already have an account with iotlab + #using another login. + + logger.debug("\r\n \t LDAP.PY make_ldap_filters_from_record \ + record %s req_ldapdict %s" + % (record, req_ldapdict)) + + for k in req_ldapdict: + req_ldap += '(' + str(k) + '=' + str(req_ldapdict[k]) + ')' + if len(req_ldapdict.keys()) >1 : + req_ldap = req_ldap[:0]+"(&"+req_ldap[0:] + size = len(req_ldap) + req_ldap = req_ldap[:(size-1)] + ')' + req_ldap[(size-1):] + else: + req_ldap = "(cn=*)" + + return req_ldap + + def make_ldap_attributes_from_record(self, record): + """ + + When adding a new user to Iotlab's LDAP, creates an attributes + dictionnary from the SFA record understandable by LDAP. Generates the + user's LDAP login.User is automatically validated (account enabled) + and described as a SFA USER FROM OUTSIDE IOTLAB. + + :param record: must contain the following keys and values: + first_name, last_name, mail, pkey (ssh key). + :type record: dict + :returns: dictionary of attributes using LDAP data structure model. + :rtype: dict + + """ + + attrs = {} + attrs['objectClass'] = ["top", "person", "inetOrgPerson", + "organizationalPerson", "posixAccount", + "shadowAccount", "systemQuotas", + "ldapPublicKey"] + + attrs['uid'] = self.LdapGenerateUniqueLogin(record) + try: + attrs['givenName'] = str(record['first_name']).lower().capitalize() + attrs['sn'] = str(record['last_name']).lower().capitalize() + attrs['cn'] = attrs['givenName'] + ' ' + attrs['sn'] + attrs['gecos'] = attrs['givenName'] + ' ' + attrs['sn'] + + except KeyError: + attrs['givenName'] = attrs['uid'] + attrs['sn'] = attrs['uid'] + attrs['cn'] = attrs['uid'] + attrs['gecos'] = attrs['uid'] + + attrs['quota'] = self.ldapUserQuotaNFS + attrs['homeDirectory'] = self.ldapUserHomePath + attrs['uid'] + attrs['loginShell'] = self.ldapShell + attrs['gidNumber'] = self.ldapUserGidNumber + attrs['uidNumber'] = self.find_max_uidNumber() + attrs['mail'] = record['mail'].lower() + try: + attrs['sshPublicKey'] = record['pkey'] + except KeyError: + attrs['sshPublicKey'] = self.get_ssh_pkey(record) + + + #Password is automatically generated because SFA user don't go + #through the Iotlab website used to register new users, + #There is no place in SFA where users can enter such information + #yet. + #If the user wants to set his own password , he must go to the Iotlab + #website. + password = self.login_pwd.generate_password() + attrs['userPassword'] = self.login_pwd.encrypt_password(password) + + #Account automatically validated (no mail request to admins) + #Set to 0 to disable the account, -1 to enable it, + attrs['shadowExpire'] = '-1' + + #Motivation field in Iotlab + attrs['description'] = 'SFA USER FROM OUTSIDE SENSLAB' + + attrs['ou'] = 'SFA' #Optional: organizational unit + #No info about those here: + attrs['l'] = 'To be defined'#Optional: Locality. + attrs['st'] = 'To be defined' #Optional: state or province (country). + + return attrs + + + + def LdapAddUser(self, record) : + """Add SFA user to LDAP if it is not in LDAP yet. + + :param record: dictionnary with the user's data. + :returns: a dictionary with the status (Fail= False, Success= True) + and the uid of the newly added user if successful, or the error + message it is not. Dict has keys bool and message in case of + failure, and bool uid in case of success. + :rtype: dict + + .. seealso:: make_ldap_filters_from_record + + """ + logger.debug(" \r\n \t LDAP LdapAddUser \r\n\r\n ================\r\n ") + user_ldap_attrs = self.make_ldap_attributes_from_record(record) + + #Check if user already in LDAP wih email, first name and last name + filter_by = self.make_ldap_filters_from_record(user_ldap_attrs) + user_exist = self.LdapSearch(filter_by) + if user_exist: + logger.warning(" \r\n \t LDAP LdapAddUser user %s %s \ + already exists" % (user_ldap_attrs['sn'], + user_ldap_attrs['mail'])) + return {'bool': False} + + #Bind to the server + result = self.conn.connect() + + if(result['bool']): + + # A dict to help build the "body" of the object + logger.debug(" \r\n \t LDAP LdapAddUser attrs %s " + % user_ldap_attrs) + + # The dn of our new entry/object + dn = 'uid=' + user_ldap_attrs['uid'] + "," + self.baseDN + + try: + ldif = modlist.addModlist(user_ldap_attrs) + logger.debug("LDAPapi.py add attrs %s \r\n ldif %s" + % (user_ldap_attrs, ldif)) + self.conn.ldapserv.add_s(dn, ldif) + + logger.info("Adding user %s login %s in LDAP" + % (user_ldap_attrs['cn'], user_ldap_attrs['uid'])) + except ldap.LDAPError, error: + logger.log_exc("LDAP Add Error %s" % error) + return {'bool': False, 'message': error} + + self.conn.close() + return {'bool': True, 'uid': user_ldap_attrs['uid']} + else: + return result + + def LdapDelete(self, person_dn): + """Deletes a person in LDAP. Uses the dn of the user. + + :param person_dn: user's ldap dn. + :type person_dn: string + :returns: dictionary with bool True if successful, bool False + and the error if not. + :rtype: dict + + """ + #Connect and bind + result = self.conn.connect() + if(result['bool']): + try: + self.conn.ldapserv.delete_s(person_dn) + self.conn.close() + return {'bool': True} + + except ldap.LDAPError, error: + logger.log_exc("LDAP Delete Error %s" % error) + return {'bool': False, 'message': error} + + def LdapDeleteUser(self, record_filter): + """Deletes a SFA person in LDAP, based on the user's hrn. + + :param record_filter: Filter to find the user to be deleted. Must + contain at least the user's email. + :type record_filter: dict + :returns: dict with bool True if successful, bool False and error + message otherwise. + :rtype: dict + + .. seealso:: LdapFindUser docstring for more info on record filter. + .. seealso:: LdapDelete for user deletion + + """ + #Find uid of the person + person = self.LdapFindUser(record_filter, []) + logger.debug("LDAPapi.py \t LdapDeleteUser record %s person %s" + % (record_filter, person)) + + if person: + dn = 'uid=' + person['uid'] + "," + self.baseDN + else: + return {'bool': False} + + result = self.LdapDelete(dn) + return result + + def LdapModify(self, dn, old_attributes_dict, new_attributes_dict): + """ Modifies a LDAP entry, replaces user's old attributes with + the new ones given. + + :param dn: user's absolute name in the LDAP hierarchy. + :param old_attributes_dict: old user's attributes. Keys must match + the ones used in the LDAP model. + :param new_attributes_dict: new user's attributes. Keys must match + the ones used in the LDAP model. + :type dn: string + :type old_attributes_dict: dict + :type new_attributes_dict: dict + :returns: dict bool True if Successful, bool False if not. + :rtype: dict + + """ + + ldif = modlist.modifyModlist(old_attributes_dict, new_attributes_dict) + # Connect and bind/authenticate + result = self.conn.connect() + if (result['bool']): + try: + self.conn.ldapserv.modify_s(dn, ldif) + self.conn.close() + return {'bool': True} + except ldap.LDAPError, error: + logger.log_exc("LDAP LdapModify Error %s" % error) + return {'bool': False} + + + def LdapModifyUser(self, user_record, new_attributes_dict): + """ + + Gets the record from one user based on the user sfa recordand changes + the attributes according to the specified new_attributes. Do not use + this if we need to modify the uid. Use a ModRDN operation instead + ( modify relative DN ). + + :param user_record: sfa user record. + :param new_attributes_dict: new user attributes, keys must be the + same as the LDAP model. + :type user_record: dict + :type new_attributes_dict: dict + :returns: bool True if successful, bool False if not. + :rtype: dict + + .. seealso:: make_ldap_filters_from_record for info on what is mandatory + in the user_record. + .. seealso:: make_ldap_attributes_from_record for the LDAP objectclass. + + """ + if user_record is None: + logger.error("LDAP \t LdapModifyUser Need user record ") + return {'bool': False} + + #Get all the attributes of the user_uid_login + #person = self.LdapFindUser(record_filter,[]) + req_ldap = self.make_ldap_filters_from_record(user_record) + person_list = self.LdapSearch(req_ldap, []) + logger.debug("LDAPapi.py \t LdapModifyUser person_list : %s" + % (person_list)) + + if person_list and len(person_list) > 1: + logger.error("LDAP \t LdapModifyUser Too many users returned") + return {'bool': False} + if person_list is None: + logger.error("LDAP \t LdapModifyUser User %s doesn't exist " + % (user_record)) + return {'bool': False} + + # The dn of our existing entry/object + #One result only from ldapSearch + person = person_list[0][1] + dn = 'uid=' + person['uid'][0] + "," + self.baseDN + + if new_attributes_dict: + old = {} + for k in new_attributes_dict: + if k not in person: + old[k] = '' + else: + old[k] = person[k] + logger.debug(" LDAPapi.py \t LdapModifyUser new_attributes %s" + % (new_attributes_dict)) + result = self.LdapModify(dn, old, new_attributes_dict) + return result + else: + logger.error("LDAP \t LdapModifyUser No new attributes given. ") + return {'bool': False} + + + def LdapMarkUserAsDeleted(self, record): + """ + + Sets shadowExpire to 0, disabling the user in LDAP. Calls LdapModifyUser + to change the shadowExpire of the user. + + :param record: the record of the user who has to be disabled. + Should contain first_name,last_name, email or mail, and if the + record is enabled or not. If the dict record does not have all of + these, must at least contain the user's email. + :type record: dict + :returns: {bool: True} if successful or {bool: False} if not + :rtype: dict + + .. seealso:: LdapModifyUser, make_ldap_attributes_from_record + """ + + new_attrs = {} + #Disable account + new_attrs['shadowExpire'] = '0' + logger.debug(" LDAPapi.py \t LdapMarkUserAsDeleted ") + ret = self.LdapModifyUser(record, new_attrs) + return ret + + def LdapResetPassword(self, record): + """Resets password for the user whose record is the parameter and + changes the corresponding entry in the LDAP. + + :param record: user's sfa record whose Ldap password must be reset. + Should contain first_name,last_name, + email or mail, and if the record is enabled or not. If the dict + record does not have all of these, must at least contain the user's + email. + :type record: dict + :returns: return value of LdapModifyUser. True if successful, False + otherwise. + + .. seealso:: LdapModifyUser, make_ldap_attributes_from_record + + """ + password = self.login_pwd.generate_password() + attrs = {} + attrs['userPassword'] = self.login_pwd.encrypt_password(password) + logger.debug("LDAP LdapResetPassword encrypt_password %s" + % (attrs['userPassword'])) + result = self.LdapModifyUser(record, attrs) + return result + + + def LdapSearch(self, req_ldap=None, expected_fields=None): + """ + Used to search directly in LDAP, by using ldap filters and return + fields. When req_ldap is None, returns all the entries in the LDAP. + + :param req_ldap: ldap style request, with appropriate filters, + example: (cn=*). + :param expected_fields: Fields in the user ldap entry that has to be + returned. If None is provided, will return 'mail', 'givenName', + 'sn', 'uid', 'sshPublicKey', 'shadowExpire'. + :type req_ldap: string + :type expected_fields: list + + .. seealso:: make_ldap_filters_from_record for req_ldap format. + + """ + result = self.conn.connect(bind=False) + if (result['bool']): + + return_fields_list = [] + if expected_fields is None: + return_fields_list = ['mail', 'givenName', 'sn', 'uid', + 'sshPublicKey', 'shadowExpire'] + else: + return_fields_list = expected_fields + #No specifc request specified, get the whole LDAP + if req_ldap is None: + req_ldap = '(cn=*)' + + logger.debug("LDAP.PY \t LdapSearch req_ldap %s \ + return_fields_list %s" \ + %(req_ldap, return_fields_list)) + + try: + msg_id = self.conn.ldapserv.search( + self.baseDN, ldap.SCOPE_SUBTREE, + req_ldap, return_fields_list) + #Get all the results matching the search from ldap in one + #shot (1 value) + result_type, result_data = \ + self.conn.ldapserv.result(msg_id, 1) + + self.conn.close() + + logger.debug("LDAP.PY \t LdapSearch result_data %s" + % (result_data)) + + return result_data + + except ldap.LDAPError, error: + logger.log_exc("LDAP LdapSearch Error %s" % error) + return [] + + else: + logger.error("LDAP.PY \t Connection Failed") + return + + def _process_ldap_info_for_all_users(self, result_data): + """Process the data of all enabled users in LDAP. + + :param result_data: Contains information of all enabled users in LDAP + and is coming from LdapSearch. + :param result_data: list + + .. seealso:: LdapSearch + + """ + results = [] + logger.debug(" LDAP.py _process_ldap_info_for_all_users result_data %s " + % (result_data)) + for ldapentry in result_data: + logger.debug(" LDAP.py _process_ldap_info_for_all_users \ + ldapentry name : %s " % (ldapentry[1]['uid'][0])) + tmpname = ldapentry[1]['uid'][0] + hrn = self.authname + "." + tmpname + + tmpemail = ldapentry[1]['mail'][0] + if ldapentry[1]['mail'][0] == "unknown": + tmpemail = None + + try: + results.append({ + 'type': 'user', + 'pkey': ldapentry[1]['sshPublicKey'][0], + #'uid': ldapentry[1]['uid'][0], + 'uid': tmpname , + 'email':tmpemail, + #'email': ldapentry[1]['mail'][0], + 'first_name': ldapentry[1]['givenName'][0], + 'last_name': ldapentry[1]['sn'][0], + #'phone': 'none', + 'serial': 'none', + 'authority': self.authname, + 'peer_authority': '', + 'pointer': -1, + 'hrn': hrn, + }) + except KeyError, error: + logger.log_exc("LDAPapi.PY \t LdapFindUser EXCEPTION %s" + % (error)) + return + + return results + + def _process_ldap_info_for_one_user(self, record, result_data): + """ + + Put the user's ldap data into shape. Only deals with one user + record and one user data from ldap. + + :param record: user record + :param result_data: Raw ldap data coming from LdapSearch + :returns: user's data dict with 'type','pkey','uid', 'email', + 'first_name' 'last_name''serial''authority''peer_authority' + 'pointer''hrn' + :type record: dict + :type result_data: list + :rtype :dict + + """ + #One entry only in the ldap data because we used a filter + #to find one user only + ldapentry = result_data[0][1] + logger.debug("LDAP.PY \t LdapFindUser ldapentry %s" % (ldapentry)) + tmpname = ldapentry['uid'][0] + + tmpemail = ldapentry['mail'][0] + if ldapentry['mail'][0] == "unknown": + tmpemail = None + + parent_hrn = None + peer_authority = None + if 'hrn' in record: + hrn = record['hrn'] + parent_hrn = get_authority(hrn) + if parent_hrn != self.authname: + peer_authority = parent_hrn + #In case the user was not imported from Iotlab LDAP + #but from another federated site, has an account in + #iotlab but currently using his hrn from federated site + #then the login is different from the one found in its hrn + if tmpname != hrn.split('.')[1]: + hrn = None + else: + hrn = None + + results = { + 'type': 'user', + 'pkey': ldapentry['sshPublicKey'], + #'uid': ldapentry[1]['uid'][0], + 'uid': tmpname, + 'email': tmpemail, + #'email': ldapentry[1]['mail'][0], + 'first_name': ldapentry['givenName'][0], + 'last_name': ldapentry['sn'][0], + #'phone': 'none', + 'serial': 'none', + 'authority': parent_hrn, + 'peer_authority': peer_authority, + 'pointer': -1, + 'hrn': hrn, + } + return results + + def LdapFindUser(self, record=None, is_user_enabled=None, + expected_fields=None): + """ + + Search a SFA user with a hrn. User should be already registered + in Iotlab LDAP. + + :param record: sfa user's record. Should contain first_name,last_name, + email or mail. If no record is provided, returns all the users found + in LDAP. + :type record: dict + :param is_user_enabled: is the user's iotlab account already valid. + :type is_user_enabled: Boolean. + :returns: LDAP entries from ldap matching the filter provided. Returns + a single entry if one filter has been given and a list of + entries otherwise. + :rtype: dict or list + + """ + custom_record = {} + if is_user_enabled: + custom_record['enabled'] = is_user_enabled + if record: + custom_record.update(record) + + req_ldap = self.make_ldap_filters_from_record(custom_record) + return_fields_list = [] + if expected_fields is None: + return_fields_list = ['mail', 'givenName', 'sn', 'uid', + 'sshPublicKey'] + else: + return_fields_list = expected_fields + + result_data = self.LdapSearch(req_ldap, return_fields_list) + logger.debug("LDAP.PY \t LdapFindUser result_data %s" % (result_data)) + + if len(result_data) == 0: + return None + #Asked for a specific user + if record is not None: + results = self._process_ldap_info_for_one_user(record, result_data) + + else: + #Asked for all users in ldap + results = self._process_ldap_info_for_all_users(result_data) + return results \ No newline at end of file diff --git a/sfa/iotlab/OARrestapi.py b/sfa/iotlab/OARrestapi.py new file mode 100644 index 00000000..a4ce7a8b --- /dev/null +++ b/sfa/iotlab/OARrestapi.py @@ -0,0 +1,990 @@ +""" +File used to handle issuing request to OAR and parse OAR's JSON responses. +Contains the following classes: +- JsonPage : handles multiple pages OAR answers. +- OARRestapi : handles issuing POST or GET requests to OAR. +- ParsingResourcesFull : dedicated to parsing OAR's answer to a get resources +full request. +- OARGETParser : handles parsing the Json answers to different GET requests. + +""" +from httplib import HTTPConnection, HTTPException, NotConnected +import json +from sfa.util.config import Config +from sfa.util.sfalogging import logger +import os.path + + +class JsonPage: + + """Class used to manipulate json pages given by OAR. + + In case the json answer from a GET request is too big to fit in one json + page, this class provides helper methods to retrieve all the pages and + store them in a list before putting them into one single json dictionary, + facilitating the parsing. + + """ + + def __init__(self): + """Defines attributes to manipulate and parse the json pages. + + """ + #All are boolean variables + self.concatenate = False + #Indicates end of data, no more pages to be loaded. + self.end = False + self.next_page = False + #Next query address + self.next_offset = None + #Json page + self.raw_json = None + + def FindNextPage(self): + """ + Gets next data page from OAR when the query's results are too big to + be transmitted in a single page. Uses the "links' item in the json + returned to check if an additionnal page has to be loaded. Updates + object attributes next_page, next_offset, and end. + + """ + if "links" in self.raw_json: + for page in self.raw_json['links']: + if page['rel'] == 'next': + self.concatenate = True + self.next_page = True + self.next_offset = "?" + page['href'].split("?")[1] + return + + if self.concatenate: + self.end = True + self.next_page = False + self.next_offset = None + + return + + #Otherwise, no next page and no concatenate, must be a single page + #Concatenate the single page and get out of here. + else: + self.next_page = False + self.concatenate = True + self.next_offset = None + return + + @staticmethod + def ConcatenateJsonPages(saved_json_list): + """ + If the json answer is too big to be contained in a single page, + all the pages have to be loaded and saved before being appended to the + first page. + + :param saved_json_list: list of all the stored pages, including the + first page. + :type saved_json_list: list + :returns: Returns a dictionary with all the pages saved in the + saved_json_list. The key of the dictionary is 'items'. + :rtype: dict + + + .. seealso:: SendRequest + .. warning:: Assumes the apilib is 0.2.10 (with the 'items' key in the + raw json dictionary) + + """ + #reset items list + + tmp = {} + tmp['items'] = [] + + for page in saved_json_list: + tmp['items'].extend(page['items']) + return tmp + + def ResetNextPage(self): + """ + Resets all the Json page attributes (next_page, next_offset, + concatenate, end). Has to be done before getting another json answer + so that the previous page status does not affect the new json load. + + """ + self.next_page = True + self.next_offset = None + self.concatenate = False + self.end = False + + +class OARrestapi: + """Class used to connect to the OAR server and to send GET and POST + requests. + + """ + + # classes attributes + + OAR_REQUEST_POST_URI_DICT = {'POST_job': {'uri': '/oarapi/jobs.json'}, + 'DELETE_jobs_id': + {'uri': '/oarapi/jobs/id.json'}, + } + + POST_FORMAT = {'json': {'content': "application/json", 'object': json}} + + #OARpostdatareqfields = {'resource' :"/nodes=", 'command':"sleep", \ + #'workdir':"/home/", 'walltime':""} + + def __init__(self, config_file='/etc/sfa/oar_config.py'): + self.oarserver = {} + self.oarserver['uri'] = None + self.oarserver['postformat'] = 'json' + + try: + execfile(config_file, self.__dict__) + + self.config_file = config_file + # path to configuration data + self.config_path = os.path.dirname(config_file) + + except IOError: + raise IOError, "Could not find or load the configuration file: %s" \ + % config_file + #logger.setLevelDebug() + self.oarserver['ip'] = self.OAR_IP + self.oarserver['port'] = self.OAR_PORT + self.jobstates = ['Terminated', 'Hold', 'Waiting', 'toLaunch', + 'toError', 'toAckReservation', 'Launching', + 'Finishing', 'Running', 'Suspended', 'Resuming', + 'Error'] + + self.parser = OARGETParser(self) + + + def GETRequestToOARRestAPI(self, request, strval=None, + next_page=None, username=None): + + """Makes a GET request to OAR. + + Fetch the uri associated with the resquest stored in + OARrequests_uri_dict, adds the username if needed and if available, adds + strval to the request uri if needed, connects to OAR and issues the GET + request. Gets the json reply. + + :param request: One of the known get requests that are keys in the + OARrequests_uri_dict. + :param strval: used when a job id has to be specified. + :param next_page: used to tell OAR to send the next page for this + Get request. Is appended to the GET uri. + :param username: used when a username has to be specified, when looking + for jobs scheduled by a particular user for instance. + + :type request: string + :type strval: integer + :type next_page: boolean + :type username: string + :returns: a json dictionary if OAR successfully processed the GET + request. + + .. seealso:: OARrequests_uri_dict + """ + self.oarserver['uri'] = \ + OARGETParser.OARrequests_uri_dict[request]['uri'] + #Get job details with username + if 'owner' in OARGETParser.OARrequests_uri_dict[request] and username: + self.oarserver['uri'] += \ + OARGETParser.OARrequests_uri_dict[request]['owner'] + username + headers = {} + data = json.dumps({}) + logger.debug("OARrestapi \tGETRequestToOARRestAPI %s" % (request)) + if strval: + self.oarserver['uri'] = self.oarserver['uri'].\ + replace("id", str(strval)) + + if next_page: + self.oarserver['uri'] += next_page + + if username: + headers['X-REMOTE_IDENT'] = username + + logger.debug("OARrestapi: \t GETRequestToOARRestAPI \ + self.oarserver['uri'] %s strval %s" + % (self.oarserver['uri'], strval)) + try: + #seems that it does not work if we don't add this + headers['content-length'] = '0' + + conn = HTTPConnection(self.oarserver['ip'], + self.oarserver['port']) + conn.request("GET", self.oarserver['uri'], data, headers) + resp = conn.getresponse() + body = resp.read() + except Exception as error: + logger.log_exc("GET_OAR_SRVR : Connection error: %s " + % (error)) + raise Exception ("GET_OAR_SRVR : Connection error %s " %(error)) + + finally: + conn.close() + + # except HTTPException, error: + # logger.log_exc("GET_OAR_SRVR : Problem with OAR server : %s " + # % (error)) + #raise ServerError("GET_OAR_SRVR : Could not reach OARserver") + if resp.status >= 400: + raise ValueError ("Response Error %s, %s" %(resp.status, + resp.reason)) + try: + js_dict = json.loads(body) + #print "\r\n \t\t\t js_dict keys" , js_dict.keys(), " \r\n", js_dict + return js_dict + + except ValueError, error: + logger.log_exc("Failed to parse Server Response: %s ERROR %s" + % (body, error)) + #raise ServerError("Failed to parse Server Response:" + js) + + + def POSTRequestToOARRestAPI(self, request, datadict, username=None): + """ Used to post a job on OAR , along with data associated + with the job. + + """ + + #first check that all params for are OK + try: + self.oarserver['uri'] = \ + self.OAR_REQUEST_POST_URI_DICT[request]['uri'] + + except KeyError: + logger.log_exc("OARrestapi \tPOSTRequestToOARRestAPI request not \ + valid") + return + if datadict and 'strval' in datadict: + self.oarserver['uri'] = self.oarserver['uri'].replace("id", \ + str(datadict['strval'])) + del datadict['strval'] + + data = json.dumps(datadict) + headers = {'X-REMOTE_IDENT':username, \ + 'content-type': self.POST_FORMAT['json']['content'], \ + 'content-length':str(len(data))} + try : + + conn = HTTPConnection(self.oarserver['ip'], \ + self.oarserver['port']) + conn.request("POST", self.oarserver['uri'], data, headers) + resp = conn.getresponse() + body = resp.read() + + except NotConnected: + logger.log_exc("POSTRequestToOARRestAPI NotConnected ERROR: \ + data %s \r\n \t\n \t\t headers %s uri %s" \ + %(data,headers,self.oarserver['uri'])) + except Exception as error: + logger.log_exc("POST_OAR_SERVER : Connection error: %s " + % (error)) + raise Exception ("POST_OAR_SERVER : Connection error %s " %(error)) + + finally: + conn.close() + + if resp.status >= 400: + raise ValueError ("Response Error %s, %s" %(resp.status, + resp.reason)) + + + try: + answer = json.loads(body) + logger.debug("POSTRequestToOARRestAPI : answer %s" % (answer)) + return answer + + except ValueError, error: + logger.log_exc("Failed to parse Server Response: error %s \ + %s" %(error)) + #raise ServerError("Failed to parse Server Response:" + answer) + + +class ParsingResourcesFull(): + """ + Class dedicated to parse the json response from a GET_resources_full from + OAR. + + """ + def __init__(self): + """ + Set the parsing dictionary. Works like a switch case, if the key is + found in the dictionary, then the associated function is called. + This is used in ParseNodes to create an usable dictionary from + the Json returned by OAR when issuing a GET resources full request. + + .. seealso:: ParseNodes + + """ + self.resources_fulljson_dict = { + 'network_address': self.AddNodeNetworkAddr, + 'site': self.AddNodeSite, + # 'radio': self.AddNodeRadio, + 'mobile': self.AddMobility, + 'x': self.AddPosX, + 'y': self.AddPosY, + 'z': self.AddPosZ, + 'archi': self.AddHardwareType, + 'state': self.AddBootState, + 'id': self.AddOarNodeId, + 'mobility_type': self.AddMobilityType, + } + + + + def AddOarNodeId(self, tuplelist, value): + """Adds Oar internal node id to the nodes' attributes. + + Appends tuple ('oar_id', node_id) to the tuplelist. Used by ParseNodes. + + .. seealso:: ParseNodes + + """ + + tuplelist.append(('oar_id', int(value))) + + + def AddNodeNetworkAddr(self, dictnode, value): + """First parsing function to be called to parse the json returned by OAR + answering a GET_resources (/oarapi/resources.json) request. + + When a new node is found in the json, this function is responsible for + creating a new entry in the dictionary for storing information on this + specific node. The key is the node network address, which is also the + node's hostname. + The value associated with the key is a tuple list.It contains all + the nodes attributes. The tuplelist will later be turned into a dict. + + :param dictnode: should be set to the OARGETParser atribute + node_dictlist. It will store the information on the nodes. + :param value: the node_id is the network_address in the raw json. + :type value: string + :type dictnode: dictionary + + .. seealso: ParseResources, ParseNodes + """ + + node_id = value + dictnode[node_id] = [('node_id', node_id),('hostname', node_id) ] + + return node_id + + def AddNodeSite(self, tuplelist, value): + """Add the site's node to the dictionary. + + + :param tuplelist: tuple list on which to add the node's site. + Contains the other node attributes as well. + :param value: value to add to the tuple list, in this case the node's + site. + :type tuplelist: list + :type value: string + + .. seealso:: AddNodeNetworkAddr + + """ + tuplelist.append(('site', str(value))) + + # def AddNodeRadio(tuplelist, value): + # """Add thenode's radio chipset type to the tuple list. + + # :param tuplelist: tuple list on which to add the node's mobility + # status. The tuplelist is the value associated with the node's + # id in the OARGETParser + # 's dictionary node_dictlist. + # :param value: name of the radio chipset on the node. + # :type tuplelist: list + # :type value: string + + # .. seealso:: AddNodeNetworkAddr + + # """ + # tuplelist.append(('radio', str(value))) + + def AddMobilityType(self, tuplelist, value): + """Adds which kind of mobility it is, train or roomba robot. + + :param tuplelist: tuple list on which to add the node's mobility status. + The tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. + :param value: tells if a node is a mobile node or not. The value is + found in the json. + + :type tuplelist: list + :type value: integer + + """ + tuplelist.append(('mobility_type', str(value))) + + + def AddMobility(self, tuplelist, value): + """Add if the node is a mobile node or not to the tuple list. + + :param tuplelist: tuple list on which to add the node's mobility status. + The tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. + :param value: tells if a node is a mobile node or not. The value is found + in the json. + + :type tuplelist: list + :type value: integer + + .. seealso:: AddNodeNetworkAddr + + """ + if value is 0: + tuplelist.append(('mobile', 'False')) + else: + tuplelist.append(('mobile', 'True')) + + + def AddPosX(self, tuplelist, value): + """Add the node's position on the x axis. + + :param tuplelist: tuple list on which to add the node's position . The + tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. + :param value: the position x. + + :type tuplelist: list + :type value: integer + + .. seealso:: AddNodeNetworkAddr + + """ + tuplelist.append(('posx', value )) + + + + def AddPosY(self, tuplelist, value): + """Add the node's position on the y axis. + + :param tuplelist: tuple list on which to add the node's position . The + tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. + :param value: the position y. + + :type tuplelist: list + :type value: integer + + .. seealso:: AddNodeNetworkAddr + + """ + tuplelist.append(('posy', value)) + + + + def AddPosZ(self, tuplelist, value): + """Add the node's position on the z axis. + + :param tuplelist: tuple list on which to add the node's position . The + tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. + :param value: the position z. + + :type tuplelist: list + :type value: integer + + .. seealso:: AddNodeNetworkAddr + + """ + + tuplelist.append(('posz', value)) + + + + def AddBootState(tself, tuplelist, value): + """Add the node's state, Alive or Suspected. + + :param tuplelist: tuple list on which to add the node's state . The + tuplelist is the value associated with the node's id in the + OARGETParser 's dictionary node_dictlist. + :param value: node's state. + + :type tuplelist: list + :type value: string + + .. seealso:: AddNodeNetworkAddr + + """ + tuplelist.append(('boot_state', str(value))) + + + def AddHardwareType(self, tuplelist, value): + """Add the node's hardware model and radio chipset type to the tuple + list. + + :param tuplelist: tuple list on which to add the node's architecture + and radio chipset type. + :param value: hardware type: radio chipset. The value contains both the + architecture and the radio chipset, separated by a colon. + :type tuplelist: list + :type value: string + + .. seealso:: AddNodeNetworkAddr + + """ + + value_list = value.split(':') + tuplelist.append(('archi', value_list[0])) + tuplelist.append(('radio', value_list[1])) + + +class OARGETParser: + """Class providing parsing methods associated to specific GET requests. + + """ + + def __init__(self, srv): + self.version_json_dict = { + 'api_version': None, 'apilib_version': None, + 'api_timezone': None, 'api_timestamp': None, 'oar_version': None} + self.config = Config() + self.interface_hrn = self.config.SFA_INTERFACE_HRN + self.timezone_json_dict = { + 'timezone': None, 'api_timestamp': None, } + #self.jobs_json_dict = { + #'total' : None, 'links' : [],\ + #'offset':None , 'items' : [], } + #self.jobs_table_json_dict = self.jobs_json_dict + #self.jobs_details_json_dict = self.jobs_json_dict + self.server = srv + self.node_dictlist = {} + + self.json_page = JsonPage() + self.parsing_resourcesfull = ParsingResourcesFull() + self.site_dict = {} + self.jobs_list = [] + self.SendRequest("GET_version") + + + def ParseVersion(self): + """Parses the OAR answer to the GET_version ( /oarapi/version.json.) + + Finds the OAR apilib version currently used. Has an impact on the json + structure returned by OAR, so the version has to be known before trying + to parse the jsons returned after a get request has been issued. + Updates the attribute version_json_dict. + + """ + + if 'oar_version' in self.json_page.raw_json: + self.version_json_dict.update( + api_version=self.json_page.raw_json['api_version'], + apilib_version=self.json_page.raw_json['apilib_version'], + api_timezone=self.json_page.raw_json['api_timezone'], + api_timestamp=self.json_page.raw_json['api_timestamp'], + oar_version=self.json_page.raw_json['oar_version']) + else: + self.version_json_dict.update( + api_version=self.json_page.raw_json['api'], + apilib_version=self.json_page.raw_json['apilib'], + api_timezone=self.json_page.raw_json['api_timezone'], + api_timestamp=self.json_page.raw_json['api_timestamp'], + oar_version=self.json_page.raw_json['oar']) + + print self.version_json_dict['apilib_version'] + + + def ParseTimezone(self): + """Get the timezone used by OAR. + + Get the timezone from the answer to the GET_timezone request. + :return: api_timestamp and api timezone. + :rype: integer, integer + + .. warning:: unused. + """ + api_timestamp = self.json_page.raw_json['api_timestamp'] + api_tz = self.json_page.raw_json['timezone'] + return api_timestamp, api_tz + + def ParseJobs(self): + """Called when a GET_jobs request has been issued to OAR. + + Corresponds to /oarapi/jobs.json uri. Currently returns the raw json + information dict. + :returns: json_page.raw_json + :rtype: dictionary + + .. warning:: Does not actually parse the information in the json. SA + 15/07/13. + + """ + self.jobs_list = [] + print " ParseJobs " + return self.json_page.raw_json + + def ParseJobsTable(self): + """In case we need to use the job table in the future. + + Associated with the GET_jobs_table : '/oarapi/jobs/table.json uri. + .. warning:: NOT USED. DOES NOTHING. + """ + print "ParseJobsTable" + + def ParseJobsDetails(self): + """Currently only returns the same json in self.json_page.raw_json. + + .. todo:: actually parse the json + .. warning:: currently, this function is not used a lot, so I have no + idea what could be useful to parse, returning the full json. NT + """ + + #logger.debug("ParseJobsDetails %s " %(self.json_page.raw_json)) + return self.json_page.raw_json + + + def ParseJobsIds(self): + """Associated with the GET_jobs_id OAR request. + + Parses the json dict (OAR answer) to the GET_jobs_id request + /oarapi/jobs/id.json. + + + :returns: dictionary whose keys are listed in the local variable + job_resources and values that are in the json dictionary returned + by OAR with the job information. + :rtype: dict + + """ + job_resources = ['wanted_resources', 'name', 'id', 'start_time', + 'state', 'owner', 'walltime', 'message'] + + # Unused variable providing the contents of the json dict returned from + # get job resources full request + job_resources_full = [ + 'launching_directory', 'links', + 'resubmit_job_id', 'owner', 'events', 'message', + 'scheduled_start', 'id', 'array_id', 'exit_code', + 'properties', 'state', 'array_index', 'walltime', + 'type', 'initial_request', 'stop_time', 'project', + 'start_time', 'dependencies', 'api_timestamp', 'submission_time', + 'reservation', 'stdout_file', 'types', 'cpuset_name', + 'name', 'wanted_resources', 'queue', 'stderr_file', 'command'] + + + job_info = self.json_page.raw_json + #logger.debug("OARESTAPI ParseJobsIds %s" %(self.json_page.raw_json)) + values = [] + try: + for k in job_resources: + values.append(job_info[k]) + return dict(zip(job_resources, values)) + + except KeyError: + logger.log_exc("ParseJobsIds KeyError ") + + + def ParseJobsIdResources(self): + """ Parses the json produced by the request + /oarapi/jobs/id/resources.json. + Returns a list of oar node ids that are scheduled for the + given job id. + + """ + job_resources = [] + for resource in self.json_page.raw_json['items']: + job_resources.append(resource['id']) + + return job_resources + + def ParseResources(self): + """ Parses the json produced by a get_resources request on oar.""" + + #logger.debug("OARESTAPI \tParseResources " ) + #resources are listed inside the 'items' list from the json + self.json_page.raw_json = self.json_page.raw_json['items'] + self.ParseNodes() + + def ParseReservedNodes(self): + """ Returns an array containing the list of the jobs scheduled + with the reserved nodes if available. + + :returns: list of job dicts, each dict containing the following keys: + t_from, t_until, resources_ids (of the reserved nodes for this job). + If the information is not available, default values will be set for + these keys. The other keys are : state, lease_id and user. + :rtype: list + + """ + + #resources are listed inside the 'items' list from the json + reservation_list = [] + job = {} + #Parse resources info + for json_element in self.json_page.raw_json['items']: + #In case it is a real reservation (not asap case) + if json_element['scheduled_start']: + job['t_from'] = json_element['scheduled_start'] + job['t_until'] = int(json_element['scheduled_start']) + \ + int(json_element['walltime']) + #Get resources id list for the job + job['resource_ids'] = [node_dict['id'] for node_dict + in json_element['resources']] + else: + job['t_from'] = "As soon as possible" + job['t_until'] = "As soon as possible" + job['resource_ids'] = ["Undefined"] + + job['state'] = json_element['state'] + job['lease_id'] = json_element['id'] + + job['user'] = json_element['owner'] + #logger.debug("OARRestapi \tParseReservedNodes job %s" %(job)) + reservation_list.append(job) + #reset dict + job = {} + return reservation_list + + def ParseRunningJobs(self): + """ Gets the list of nodes currently in use from the attributes of the + running jobs. + + :returns: list of hostnames, the nodes that are currently involved in + running jobs. + :rtype: list + + + """ + logger.debug("OARESTAPI \tParseRunningJobs_________________ ") + #resources are listed inside the 'items' list from the json + nodes = [] + for job in self.json_page.raw_json['items']: + for node in job['nodes']: + nodes.append(node['network_address']) + return nodes + + def ChangeRawJsonDependingOnApilibVersion(self): + """ + Check if the OAR apilib version is different from 0.2.10, in which case + the Json answer is also dict instead as a plain list. + + .. warning:: the whole code is assuming the json contains a 'items' key + .. seealso:: ConcatenateJsonPages, ParseJobs, ParseReservedNodes, + ParseJobsIdResources, ParseResources, ParseRunningJobs + .. todo:: Clean the whole code. Either suppose the apilib will always + provide the 'items' key, or handle different options. + """ + + if self.version_json_dict['apilib_version'] != "0.2.10": + self.json_page.raw_json = self.json_page.raw_json['items'] + + def ParseDeleteJobs(self): + """ No need to parse anything in this function.A POST + is done to delete the job. + + """ + return + + def ParseResourcesFull(self): + """ This method is responsible for parsing all the attributes + of all the nodes returned by OAR when issuing a get resources full. + The information from the nodes and the sites are separated. + Updates the node_dictlist so that the dictionnary of the platform's + nodes is available afterwards. + + :returns: node_dictlist, a list of dictionaries about the nodes and + their properties. + :rtype: list + + """ + logger.debug("OARRESTAPI ParseResourcesFull___________ ") + #print self.json_page.raw_json[1] + #resources are listed inside the 'items' list from the json + self.ChangeRawJsonDependingOnApilibVersion() + self.ParseNodes() + self.ParseSites() + return self.node_dictlist + + def ParseResourcesFullSites(self): + """ Called by GetSites which is unused. + Originally used to get information from the sites, with for each site + the list of nodes it has, along with their properties. + + :return: site_dict, dictionary of sites + :rtype: dict + + .. warning:: unused + .. seealso:: GetSites (IotlabTestbedAPI) + + """ + self.ChangeRawJsonDependingOnApilibVersion() + self.ParseNodes() + self.ParseSites() + return self.site_dict + + + def ParseNodes(self): + """ Parse nodes properties from OAR + Put them into a dictionary with key = node id and value is a dictionary + of the node properties and properties'values. + + """ + node_id = None + _resources_fulljson_dict = \ + self.parsing_resourcesfull.resources_fulljson_dict + keys = _resources_fulljson_dict.keys() + keys.sort() + + for dictline in self.json_page.raw_json: + node_id = None + # dictionary is empty and/or a new node has to be inserted + node_id = _resources_fulljson_dict['network_address']( + self.node_dictlist, dictline['network_address']) + for k in keys: + if k in dictline: + if k == 'network_address': + continue + + _resources_fulljson_dict[k]( + self.node_dictlist[node_id], dictline[k]) + + #The last property has been inserted in the property tuple list, + #reset node_id + #Turn the property tuple list (=dict value) into a dictionary + self.node_dictlist[node_id] = dict(self.node_dictlist[node_id]) + node_id = None + + @staticmethod + def iotlab_hostname_to_hrn(root_auth, hostname): + """ + Transforms a node hostname into a SFA hrn. + + :param root_auth: Name of the root authority of the SFA server. In + our case, it is set to iotlab. + :param hostname: node's hotname, given by OAR. + :type root_auth: string + :type hostname: string + :returns: inserts the root_auth and '.' before the hostname. + :rtype: string + + """ + return root_auth + '.' + hostname + + def ParseSites(self): + """ Returns a list of dictionnaries containing the sites' attributes.""" + + nodes_per_site = {} + config = Config() + #logger.debug(" OARrestapi.py \tParseSites self.node_dictlist %s"\ + #%(self.node_dictlist)) + # Create a list of nodes per site_id + for node_id in self.node_dictlist: + node = self.node_dictlist[node_id] + + if node['site'] not in nodes_per_site: + nodes_per_site[node['site']] = [] + nodes_per_site[node['site']].append(node['node_id']) + else: + if node['node_id'] not in nodes_per_site[node['site']]: + nodes_per_site[node['site']].append(node['node_id']) + + #Create a site dictionary whose key is site_login_base + # (name of the site) and value is a dictionary of properties, + # including the list of the node_ids + for node_id in self.node_dictlist: + node = self.node_dictlist[node_id] + node.update({'hrn': self.iotlab_hostname_to_hrn(self.interface_hrn, + node['hostname'])}) + self.node_dictlist.update({node_id: node}) + + if node['site'] not in self.site_dict: + self.site_dict[node['site']] = { + 'site': node['site'], + 'node_ids': nodes_per_site[node['site']], + 'latitude': "48.83726", + 'longitude': "- 2.10336", + 'name': config.SFA_REGISTRY_ROOT_AUTH, + 'pcu_ids': [], 'max_slices': None, + 'ext_consortium_id': None, + 'max_slivers': None, 'is_public': True, + 'peer_site_id': None, + 'abbreviated_name': "iotlab", 'address_ids': [], + 'url': "https://portal.senslab.info", 'person_ids': [], + 'site_tag_ids': [], 'enabled': True, 'slice_ids': [], + 'date_created': None, 'peer_id': None + } + + OARrequests_uri_dict = { + 'GET_version': + {'uri': '/oarapi/version.json', 'parse_func': ParseVersion}, + + 'GET_timezone': + {'uri': '/oarapi/timezone.json', 'parse_func': ParseTimezone}, + + 'GET_jobs': + {'uri': '/oarapi/jobs.json', 'parse_func': ParseJobs}, + + 'GET_jobs_id': + {'uri': '/oarapi/jobs/id.json', 'parse_func': ParseJobsIds}, + + 'GET_jobs_id_resources': + {'uri': '/oarapi/jobs/id/resources.json', + 'parse_func': ParseJobsIdResources}, + + 'GET_jobs_table': + {'uri': '/oarapi/jobs/table.json', 'parse_func': ParseJobsTable}, + + 'GET_jobs_details': + {'uri': '/oarapi/jobs/details.json', 'parse_func': ParseJobsDetails}, + + 'GET_reserved_nodes': + {'uri': + '/oarapi/jobs/details.json?state=Running,Waiting,Launching', + 'owner': '&user=', 'parse_func': ParseReservedNodes}, + + 'GET_running_jobs': + {'uri': '/oarapi/jobs/details.json?state=Running', + 'parse_func': ParseRunningJobs}, + + 'GET_resources_full': + {'uri': '/oarapi/resources/full.json', + 'parse_func': ParseResourcesFull}, + + 'GET_sites': + {'uri': '/oarapi/resources/full.json', + 'parse_func': ParseResourcesFullSites}, + + 'GET_resources': + {'uri': '/oarapi/resources.json', 'parse_func': ParseResources}, + + 'DELETE_jobs_id': + {'uri': '/oarapi/jobs/id.json', 'parse_func': ParseDeleteJobs}} + + + def SendRequest(self, request, strval=None, username=None): + """ Connects to OAR , sends the valid GET requests and uses + the appropriate json parsing functions. + + :returns: calls to the appropriate parsing function, associated with the + GET request + :rtype: depends on the parsing function called. + + .. seealso:: OARrequests_uri_dict + """ + save_json = None + + self.json_page.ResetNextPage() + save_json = [] + + if request in self.OARrequests_uri_dict: + while self.json_page.next_page: + self.json_page.raw_json = self.server.GETRequestToOARRestAPI( + request, + strval, + self.json_page.next_offset, + username) + self.json_page.FindNextPage() + if self.json_page.concatenate: + save_json.append(self.json_page.raw_json) + + if self.json_page.concatenate and self.json_page.end: + self.json_page.raw_json = \ + self.json_page.ConcatenateJsonPages(save_json) + + return self.OARrequests_uri_dict[request]['parse_func'](self) + else: + logger.error("OARRESTAPI OARGetParse __init__ : ERROR_REQUEST " + % (request)) diff --git a/sfa/iotlab/__init__.py b/sfa/iotlab/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sfa/iotlab/iotlabaggregate.py b/sfa/iotlab/iotlabaggregate.py new file mode 100644 index 00000000..bece8bb4 --- /dev/null +++ b/sfa/iotlab/iotlabaggregate.py @@ -0,0 +1,471 @@ +""" +File providing methods to generate valid RSpecs for the Iotlab testbed. +Contains methods to get information on slice, slivers, nodes and leases, +formatting them and turn it into a RSpec. +""" +from sfa.util.xrn import hrn_to_urn, urn_to_hrn, get_authority + +from sfa.rspecs.rspec import RSpec +#from sfa.rspecs.elements.location import Location +from sfa.rspecs.elements.hardware_type import HardwareType +from sfa.rspecs.elements.login import Login +from sfa.rspecs.elements.services import Services +from sfa.rspecs.elements.sliver import Sliver +from sfa.rspecs.elements.lease import Lease +from sfa.rspecs.elements.granularity import Granularity +from sfa.rspecs.version_manager import VersionManager + +from sfa.rspecs.elements.versions.iotlabv1Node import IotlabPosition, \ + IotlabNode, IotlabLocation + +from sfa.util.sfalogging import logger +from sfa.util.xrn import Xrn + + +def iotlab_xrn_to_hostname(xrn): + """Returns a node's hostname from its xrn. + :param xrn: The nodes xrn identifier. + :type xrn: Xrn (from sfa.util.xrn) + + :returns: node's hostname. + :rtype: string + + """ + return Xrn.unescape(Xrn(xrn=xrn, type='node').get_leaf()) + + +def iotlab_xrn_object(root_auth, hostname): + """Creates a valid xrn object from the node's hostname and the authority + of the SFA server. + + :param hostname: the node's hostname. + :param root_auth: the SFA root authority. + :type hostname: string + :type root_auth: string + + :returns: the iotlab node's xrn + :rtype: Xrn + + """ + return Xrn('.'.join([root_auth, Xrn.escape(hostname)]), type='node') + + +class IotlabAggregate: + """Aggregate manager class for Iotlab. """ + + sites = {} + nodes = {} + api = None + interfaces = {} + links = {} + node_tags = {} + + prepared = False + + user_options = {} + + def __init__(self, driver): + self.driver = driver + + def get_slice_and_slivers(self, slice_xrn, login=None): + """ + Get the slices and the associated leases if any from the iotlab + testbed. One slice can have mutliple leases. + For each slice, get the nodes in the associated lease + and create a sliver with the necessary info and insert it into the + sliver dictionary, keyed on the node hostnames. + Returns a dict of slivers based on the sliver's node_id. + Called by get_rspec. + + + :param slice_xrn: xrn of the slice + :param login: user's login on iotlab ldap + + :type slice_xrn: string + :type login: string + :returns: a list of slices dict and a list of Sliver object + :rtype: (list, list) + + .. note: There is no real slivers in iotlab, only leases. The goal + is to be consistent with the SFA standard. + + """ + slivers = {} + sfa_slice = None + if slice_xrn is None: + return (sfa_slice, slivers) + slice_urn = hrn_to_urn(slice_xrn, 'slice') + slice_hrn, _ = urn_to_hrn(slice_xrn) + slice_name = slice_hrn + + # GetSlices always returns a list, even if there is only one element + slices = self.driver.iotlab_api.GetSlices(slice_filter=str(slice_name), + slice_filter_type='slice_hrn', + login=login) + + logger.debug("IotlabAggregate api \tget_slice_and_slivers \ + slice_hrn %s \r\n slices %s self.driver.hrn %s" + % (slice_hrn, slices, self.driver.hrn)) + if slices == []: + return (sfa_slice, slivers) + + # sort slivers by node id , if there is a job + #and therefore, node allocated to this slice + # for sfa_slice in slices: + sfa_slice = slices[0] + try: + node_ids_list = sfa_slice['node_ids'] + except KeyError: + logger.log_exc("IOTLABAGGREGATE \t \ + get_slice_and_slivers No nodes in the slice \ + - KeyError ") + node_ids_list = [] + # continue + + for node in node_ids_list: + sliver_xrn = Xrn(slice_urn, type='sliver', id=node) + sliver_xrn.set_authority(self.driver.hrn) + sliver = Sliver({'sliver_id': sliver_xrn.urn, + 'name': sfa_slice['hrn'], + 'type': 'iotlab-node', + 'tags': []}) + + slivers[node] = sliver + + #Add default sliver attribute : + #connection information for iotlab + # if get_authority(sfa_slice['hrn']) == self.driver.iotlab_api.root_auth: + # tmp = sfa_slice['hrn'].split('.') + # ldap_username = tmp[1].split('_')[0] + # ssh_access = None + # slivers['default_sliver'] = {'ssh': ssh_access, + # 'login': ldap_username} + # look in ldap: + ldap_username = self.find_ldap_username_from_slice(sfa_slice) + + if ldap_username is not None: + ssh_access = None + slivers['default_sliver'] = {'ssh': ssh_access, + 'login': ldap_username} + + + logger.debug("IOTLABAGGREGATE api get_slice_and_slivers slivers %s " + % (slivers)) + return (slices, slivers) + + def find_ldap_username_from_slice(self, sfa_slice): + researchers = [sfa_slice['reg_researchers'][0].__dict__] + # look in ldap: + ldap_username = None + ret = self.driver.iotlab_api.GetPersons(researchers) + if len(ret) != 0: + ldap_username = ret[0]['uid'] + + return ldap_username + + + def get_nodes(self, slices=None, slivers=[], options=None): + """Returns the nodes in the slice using the rspec format, with all the + nodes' properties. + + Fetch the nodes ids in the slices dictionary and get all the nodes + properties from OAR. Makes a rspec dicitonary out of this and returns + it. If the slice does not have any job running or scheduled, that is + it has no reserved nodes, then returns an empty list. + + :param slices: list of slices (record dictionaries) + :param slivers: the list of slivers in all the slices + :type slices: list of dicts + :type slivers: list of Sliver object (dictionaries) + :returns: An empty list if the slice has no reserved nodes, a rspec + list with all the nodes and their properties (a dict per node) + otherwise. + :rtype: list + + .. seealso:: get_slice_and_slivers + + """ + # NT: the semantic of this function is not clear to me : + # if slice is not defined, then all the nodes should be returned + # if slice is defined, we should return only the nodes that + # are part of this slice + # but what is the role of the slivers parameter ? + # So i assume that slice['node_ids'] will be the same as slivers for us + slice_nodes_list = [] + if slices is not None: + for one_slice in slices: + try: + slice_nodes_list = one_slice['node_ids'] + # if we are dealing with a slice that has no node just + # return an empty list. In iotlab a slice can have multiple + # jobs scheduled, so it either has at least one lease or + # not at all. + except KeyError: + return [] + + # get the granularity in second for the reservation system + grain = self.driver.iotlab_api.GetLeaseGranularity() + + nodes = self.driver.iotlab_api.GetNodes() + + nodes_dict = {} + + #if slices, this means we got to list all the nodes given to this slice + # Make a list of all the nodes in the slice before getting their + #attributes + rspec_nodes = [] + + logger.debug("IOTLABAGGREGATE api get_nodes slices %s " + % (slices)) + + + reserved_nodes = self.driver.iotlab_api.GetNodesCurrentlyInUse() + logger.debug("IOTLABAGGREGATE api get_nodes slice_nodes_list %s " + % (slice_nodes_list)) + for node in nodes: + nodes_dict[node['node_id']] = node + if slice_nodes_list == [] or node['hostname'] in slice_nodes_list: + + rspec_node = IotlabNode() + # xxx how to retrieve site['login_base'] + #site_id=node['site_id'] + #site=sites_dict[site_id] + + rspec_node['mobile'] = node['mobile'] + rspec_node['archi'] = node['archi'] + rspec_node['radio'] = node['radio'] + + iotlab_xrn = iotlab_xrn_object(self.driver.iotlab_api.root_auth, + node['hostname']) + rspec_node['component_id'] = iotlab_xrn.urn + rspec_node['component_name'] = node['hostname'] + rspec_node['component_manager_id'] = \ + hrn_to_urn(self.driver.iotlab_api.root_auth, + 'authority+sa') + + # Iotlab's nodes are federated : there is only one authority + # for all Iotlab sites, registered in SFA. + # Removing the part including the site + # in authority_id SA 27/07/12 + rspec_node['authority_id'] = rspec_node['component_manager_id'] + + # do not include boot state ( element) + #in the manifest rspec + + + rspec_node['boot_state'] = node['boot_state'] + if node['hostname'] in reserved_nodes: + rspec_node['boot_state'] = "Reserved" + rspec_node['exclusive'] = 'true' + rspec_node['hardware_types'] = [HardwareType({'name': \ + 'iotlab-node'})] + + + location = IotlabLocation({'country':'France', 'site': \ + node['site']}) + rspec_node['location'] = location + + + position = IotlabPosition() + for field in position : + try: + position[field] = node[field] + except KeyError, error : + logger.log_exc("IOTLABAGGREGATE\t get_nodes \ + position %s "% (error)) + + rspec_node['position'] = position + #rspec_node['interfaces'] = [] + + # Granularity + granularity = Granularity({'grain': grain}) + rspec_node['granularity'] = granularity + rspec_node['tags'] = [] + if node['hostname'] in slivers: + # add sliver info + sliver = slivers[node['hostname']] + rspec_node['sliver_id'] = sliver['sliver_id'] + rspec_node['client_id'] = node['hostname'] + rspec_node['slivers'] = [sliver] + + # slivers always provide the ssh service + login = Login({'authentication': 'ssh-keys', \ + 'hostname': node['hostname'], 'port':'22', \ + 'username': sliver['name']}) + service = Services({'login': login}) + rspec_node['services'] = [service] + rspec_nodes.append(rspec_node) + + return (rspec_nodes) + + def get_all_leases(self, ldap_username): + """ + + Get list of lease dictionaries which all have the mandatory keys + ('lease_id', 'hostname', 'site_id', 'name', 'start_time', 'duration'). + All the leases running or scheduled are returned. + + :param ldap_username: if ldap uid is not None, looks for the leases + belonging to this user. + :type ldap_username: string + :returns: rspec lease dictionary with keys lease_id, component_id, + slice_id, start_time, duration. + :rtype: dict + + .. note::There is no filtering of leases within a given time frame. + All the running or scheduled leases are returned. options + removed SA 15/05/2013 + + + """ + + #now = int(time.time()) + #lease_filter = {'clip': now } + + #if slice_record: + #lease_filter.update({'name': slice_record['name']}) + + #leases = self.driver.iotlab_api.GetLeases(lease_filter) + + logger.debug("IOTLABAGGREGATE get_all_leases ldap_username %s " + % (ldap_username)) + leases = self.driver.iotlab_api.GetLeases(login=ldap_username) + grain = self.driver.iotlab_api.GetLeaseGranularity() + # site_ids = [] + rspec_leases = [] + for lease in leases: + #as many leases as there are nodes in the job + for node in lease['reserved_nodes']: + rspec_lease = Lease() + rspec_lease['lease_id'] = lease['lease_id'] + #site = node['site_id'] + iotlab_xrn = iotlab_xrn_object(self.driver.iotlab_api.root_auth, + node) + rspec_lease['component_id'] = iotlab_xrn.urn + #rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn,\ + #site, node['hostname']) + try: + rspec_lease['slice_id'] = lease['slice_id'] + except KeyError: + #No info on the slice used in testbed_xp table + pass + rspec_lease['start_time'] = lease['t_from'] + rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) \ + / grain + rspec_leases.append(rspec_lease) + return rspec_leases + + def get_rspec(self, slice_xrn=None, login=None, version=None, + options=None): + """ + Returns xml rspec: + - a full advertisement rspec with the testbed resources if slice_xrn is + not specified.If a lease option is given, also returns the leases + scheduled on the testbed. + - a manifest Rspec with the leases and nodes in slice's leases if + slice_xrn is not None. + + :param slice_xrn: srn of the slice + :type slice_xrn: string + :param login: user'uid (ldap login) on iotlab + :type login: string + :param version: can be set to sfa or iotlab + :type version: RSpecVersion + :param options: used to specify if the leases should also be included in + the returned rspec. + :type options: dict + + :returns: Xml Rspec. + :rtype: XML + + + """ + + ldap_username= None + rspec = None + version_manager = VersionManager() + version = version_manager.get_version(version) + logger.debug("IotlabAggregate \t get_rspec ***version %s \ + version.type %s version.version %s options %s \r\n" + % (version, version.type, version.version, options)) + + if slice_xrn is None: + rspec_version = version_manager._get_version(version.type, + version.version, 'ad') + + else: + rspec_version = version_manager._get_version( + version.type, version.version, 'manifest') + + slices, slivers = self.get_slice_and_slivers(slice_xrn, login) + if slice_xrn and slices is not None: + #Get user associated with this slice + #for one_slice in slices : + ldap_username = self.find_ldap_username_from_slice(slices[0]) + # ldap_username = slices[0]['reg_researchers'][0].__dict__['hrn'] + # # ldap_username = slices[0]['user'] + # tmp = ldap_username.split('.') + # ldap_username = tmp[1] + logger.debug("IotlabAggregate \tget_rspec **** \ + LDAP USERNAME %s \r\n" \ + % (ldap_username)) + #at this point sliver may be empty if no iotlab job + #is running for this user/slice. + rspec = RSpec(version=rspec_version, user_options=options) + + logger.debug("\r\n \r\n IotlabAggregate \tget_rspec *** \ + slice_xrn %s slices %s\r\n \r\n" + % (slice_xrn, slices)) + + if options is not None: + lease_option = options['list_leases'] + else: + #If no options are specified, at least print the resources + lease_option = 'all' + #if slice_xrn : + #lease_option = 'all' + + if lease_option in ['all', 'resources']: + #if not options.get('list_leases') or options.get('list_leases') + #and options['list_leases'] != 'leases': + nodes = self.get_nodes(slices, slivers) + logger.debug("\r\n") + logger.debug("IotlabAggregate \t lease_option %s \ + get rspec ******* nodes %s" + % (lease_option, nodes)) + + sites_set = set([node['location']['site'] for node in nodes]) + + #In case creating a job, slice_xrn is not set to None + rspec.version.add_nodes(nodes) + if slice_xrn and slices is not None: + # #Get user associated with this slice + # #for one_slice in slices : + # ldap_username = slices[0]['reg_researchers'] + # # ldap_username = slices[0]['user'] + # tmp = ldap_username.split('.') + # ldap_username = tmp[1] + # # ldap_username = tmp[1].split('_')[0] + + logger.debug("IotlabAggregate \tget_rspec **** \ + version type %s ldap_ user %s \r\n" \ + % (version.type, ldap_username)) + if version.type == "Iotlab": + rspec.version.add_connection_information( + ldap_username, sites_set) + + default_sliver = slivers.get('default_sliver', []) + if default_sliver and len(nodes) is not 0: + #default_sliver_attribs = default_sliver.get('tags', []) + logger.debug("IotlabAggregate \tget_rspec **** \ + default_sliver%s \r\n" % (default_sliver)) + for attrib in default_sliver: + rspec.version.add_default_sliver_attribute( + attrib, default_sliver[attrib]) + + if lease_option in ['all','leases']: + leases = self.get_all_leases(ldap_username) + rspec.version.add_leases(leases) + logger.debug("IotlabAggregate \tget_rspec **** \ + FINAL RSPEC %s \r\n" % (rspec.toxml())) + return rspec.toxml() diff --git a/sfa/senslab/slabapi.py b/sfa/iotlab/iotlabapi.py similarity index 54% rename from sfa/senslab/slabapi.py rename to sfa/iotlab/iotlabapi.py index 5f11006b..f50441d3 100644 --- a/sfa/senslab/slabapi.py +++ b/sfa/iotlab/iotlabapi.py @@ -1,3 +1,9 @@ +""" +File containing the IotlabTestbedAPI, used to interact with nodes, users, +slices, leases and keys, as well as the dedicated iotlab database and table, +holding information about which slice is running which job. + +""" from datetime import datetime from sfa.util.sfalogging import logger @@ -5,63 +11,67 @@ from sfa.util.sfalogging import logger from sfa.storage.alchemy import dbsession from sqlalchemy.orm import joinedload from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey -from sfa.senslab.slabpostgres import slab_dbsession, SenslabXP - -from sfa.senslab.OARrestapi import OARrestapi -from sfa.senslab.LDAPapi import LDAPapi +from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB, LeaseTableXP +from sfa.iotlab.OARrestapi import OARrestapi +from sfa.iotlab.LDAPapi import LDAPapi from sfa.util.xrn import Xrn, hrn_to_urn, get_authority from sfa.trust.certificate import Keypair, convert_public_key from sfa.trust.gid import create_uuid from sfa.trust.hierarchy import Hierarchy - -from sfa.senslab.slabaggregate import slab_xrn_object -class SlabTestbedAPI(): +from sfa.iotlab.iotlabaggregate import iotlab_xrn_object + +class IotlabTestbedAPI(): """ Class enabled to use LDAP and OAR api calls. """ - + + _MINIMUM_DURATION = 10 # 10 units of granularity 60 s, 10 mins + def __init__(self, config): """Creates an instance of OARrestapi and LDAPapi which will be used to issue calls to OAR or LDAP methods. - Set the time format and the testbed granularity used for OAR - reservation and leases. - + Set the time format and the testbed granularity used for OAR + reservation and leases. + :param config: configuration object from sfa.util.config - :type config: Config object + :type config: Config object """ + self.iotlab_db = TestbedAdditionalSfaDB(config) self.oar = OARrestapi() self.ldap = LDAPapi() self.time_format = "%Y-%m-%d %H:%M:%S" self.root_auth = config.SFA_REGISTRY_ROOT_AUTH - self.grain = 1 # 10 mins lease minimum + self.grain = 60 # 10 mins lease minimum, 60 sec granularity #import logging, logging.handlers #from sfa.util.sfalogging import _SfaLogger #sql_logger = _SfaLogger(loggername = 'sqlalchemy.engine', \ #level=logging.DEBUG) return - - + + @staticmethod + def GetMinExperimentDurationInGranularity(): + """ Returns the minimum allowed duration for an experiment on the + testbed. In seconds. + + """ + return IotlabTestbedAPI._MINIMUM_DURATION + @staticmethod - def GetMinExperimentDurationInSec(): - return 600 - - @staticmethod - def GetPeers (peer_filter=None ): + def GetPeers(peer_filter=None ): """ Gathers registered authorities in SFA DB and looks for specific peer - if peer_filter is specified. + if peer_filter is specified. :param peer_filter: name of the site authority looked for. :type peer_filter: string - :return: list of records. - + :returns: list of records. + """ existing_records = {} existing_hrns_by_types = {} - logger.debug("SLABDRIVER \tGetPeers peer_filter %s, \ - " %(peer_filter)) + logger.debug("IOTLAB_API \tGetPeers peer_filter %s " % (peer_filter)) all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all() - + for record in all_records: existing_records[(record.hrn, record.type)] = record if record.type not in existing_hrns_by_types: @@ -69,71 +79,71 @@ class SlabTestbedAPI(): else: existing_hrns_by_types[record.type].append(record.hrn) - - logger.debug("SLABDRIVER \tGetPeer\texisting_hrns_by_types %s "\ - %( existing_hrns_by_types)) - records_list = [] - - try: + logger.debug("IOTLAB_API \tGetPeer\texisting_hrns_by_types %s " + % (existing_hrns_by_types)) + records_list = [] + + try: if peer_filter: - records_list.append(existing_records[(peer_filter,'authority')]) - else : + records_list.append(existing_records[(peer_filter, + 'authority')]) + else: for hrn in existing_hrns_by_types['authority']: - records_list.append(existing_records[(hrn,'authority')]) - - logger.debug("SLABDRIVER \tGetPeer \trecords_list %s " \ - %(records_list)) + records_list.append(existing_records[(hrn, 'authority')]) + + logger.debug("IOTLAB_API \tGetPeer \trecords_list %s " + % (records_list)) except KeyError: pass - + return_records = records_list - logger.debug("SLABDRIVER \tGetPeer return_records %s " \ - %(return_records)) + logger.debug("IOTLAB_API \tGetPeer return_records %s " + % (return_records)) return return_records - - - #TODO : Handling OR request in make_ldap_filters_from_records - #instead of the for loop + #TODO : Handling OR request in make_ldap_filters_from_records + #instead of the for loop #over the records' list def GetPersons(self, person_filter=None): """ - Get the enabled users and their properties from Senslab LDAP. + Get the enabled users and their properties from Iotlab LDAP. If a filter is specified, looks for the user whose properties match the filter, otherwise returns the whole enabled users'list. - :param person_filter: Must be a list of dictionnaries - with users properties when not set to None. - :param person_filter: list of dict - :return:Returns a list of users whose accounts are enabled - found in ldap. + + :param person_filter: Must be a list of dictionnaries with users + properties when not set to None. + :type person_filter: list of dict + + :returns: Returns a list of users whose accounts are enabled + found in ldap. :rtype: list of dicts - + """ - logger.debug("SLABDRIVER \tGetPersons person_filter %s" \ - %(person_filter)) + logger.debug("IOTLAB_API \tGetPersons person_filter %s" + % (person_filter)) person_list = [] if person_filter and isinstance(person_filter, list): #If we are looking for a list of users (list of dict records) #Usually the list contains only one user record for searched_attributes in person_filter: - - #Get only enabled user accounts in senslab LDAP : + + #Get only enabled user accounts in iotlab LDAP : #add a filter for make_ldap_filters_from_record - person = self.ldap.LdapFindUser(searched_attributes, \ - is_user_enabled=True) + person = self.ldap.LdapFindUser(searched_attributes, + is_user_enabled=True) #If a person was found, append it to the list if person: person_list.append(person) - + #If the list is empty, return None if len(person_list) is 0: person_list = None - + else: - #Get only enabled user accounts in senslab LDAP : + #Get only enabled user accounts in iotlab LDAP : #add a filter for make_ldap_filters_from_record - person_list = self.ldap.LdapFindUser(is_user_enabled=True) + person_list = self.ldap.LdapFindUser(is_user_enabled=True) return person_list @@ -145,235 +155,245 @@ class SlabTestbedAPI(): #SendRequest("GET_timezone") #return server_timestamp, server_tz - def DeleteJobs(self, job_id, username): - - """ Deletes the job with the specified job_id and username on OAR by - posting a delete request to OAR. - + def DeleteJobs(self, job_id, username): + """ + + Deletes the job with the specified job_id and username on OAR by + posting a delete request to OAR. + :param job_id: job id in OAR. - :param username: user's senslab login in LDAP. - :type job_id:integer + :param username: user's iotlab login in LDAP. + :type job_id: integer :type username: string - - :return: dictionary with the job id and if delete has been successful - (True) or no (False) + + :returns: dictionary with the job id and if delete has been successful + (True) or no (False) :rtype: dict - """ - logger.debug("SLABDRIVER \tDeleteJobs jobid %s username %s "\ - %(job_id, username)) + + """ + logger.debug("IOTLAB_API \tDeleteJobs jobid %s username %s " + % (job_id, username)) if not job_id or job_id is -1: return reqdict = {} reqdict['method'] = "delete" reqdict['strval'] = str(job_id) - - answer = self.oar.POSTRequestToOARRestAPI('DELETE_jobs_id', \ - reqdict,username) + answer = self.oar.POSTRequestToOARRestAPI('DELETE_jobs_id', + reqdict, username) if answer['status'] == 'Delete request registered': - ret = {job_id : True } + ret = {job_id: True} else: - ret = {job_id :False } - logger.debug("SLABDRIVER \tDeleteJobs jobid %s \r\n answer %s \ - username %s" %(job_id, answer, username)) + ret = {job_id: False} + logger.debug("IOTLAB_API \tDeleteJobs jobid %s \r\n answer %s \ + username %s" % (job_id, answer, username)) return ret - - + + ##TODO : Unused GetJobsId ? SA 05/07/12 #def GetJobsId(self, job_id, username = None ): #""" - #Details about a specific job. - #Includes details about submission time, jot type, state, events, + #Details about a specific job. + #Includes details about submission time, jot type, state, events, #owner, assigned ressources, walltime etc... - + #""" #req = "GET_jobs_id" #node_list_k = 'assigned_network_address' - ##Get job info from OAR + ##Get job info from OAR #job_info = self.oar.parser.SendRequest(req, job_id, username) - #logger.debug("SLABDRIVER \t GetJobsId %s " %(job_info)) + #logger.debug("IOTLAB_API \t GetJobsId %s " %(job_info)) #try: #if job_info['state'] == 'Terminated': - #logger.debug("SLABDRIVER \t GetJobsId job %s TERMINATED"\ + #logger.debug("IOTLAB_API \t GetJobsId job %s TERMINATED"\ #%(job_id)) #return None #if job_info['state'] == 'Error': - #logger.debug("SLABDRIVER \t GetJobsId ERROR message %s "\ + #logger.debug("IOTLAB_API \t GetJobsId ERROR message %s "\ #%(job_info)) #return None - + #except KeyError: - #logger.error("SLABDRIVER \tGetJobsId KeyError") - #return None - + #logger.error("IOTLAB_API \tGetJobsId KeyError") + #return None + #parsed_job_info = self.get_info_on_reserved_nodes(job_info, \ #node_list_k) - ##Replaces the previous entry + ##Replaces the previous entry ##"assigned_network_address" / "reserved_resources" ##with "node_ids" #job_info.update({'node_ids':parsed_job_info[node_list_k]}) #del job_info[node_list_k] - #logger.debug(" \r\nSLABDRIVER \t GetJobsId job_info %s " %(job_info)) + #logger.debug(" \r\nIOTLAB_API \t GetJobsId job_info %s " %(job_info)) #return job_info - + def GetJobsResources(self, job_id, username = None): - """ Gets the list of nodes associated with the job_id and username - if provided. - Transforms the senslab hostnames to the corresponding + """ Gets the list of nodes associated with the job_id and username + if provided. + Transforms the iotlab hostnames to the corresponding SFA nodes hrns. - Rertuns dict key :'node_ids' , value : hostnames list + Rertuns dict key :'node_ids' , value : hostnames list :param username: user's LDAP login :paran job_id: job's OAR identifier. :type username: string :type job_id: integer - - :return: dicionary with nodes' hostnames belonging to the job. + + :returns: dicionary with nodes' hostnames belonging to the job. :rtype: dict + .. warning: Unused. SA 16/10/13 """ req = "GET_jobs_id_resources" - - - #Get job resources list from OAR + + + #Get job resources list from OAR node_id_list = self.oar.parser.SendRequest(req, job_id, username) - logger.debug("SLABDRIVER \t GetJobsResources %s " %(node_id_list)) - + logger.debug("IOTLAB_API \t GetJobsResources %s " %(node_id_list)) + hostname_list = \ self.__get_hostnames_from_oar_node_ids(node_id_list) - - #Replaces the previous entry "assigned_network_address" / + + #Replaces the previous entry "assigned_network_address" / #"reserved_resources" with "node_ids" job_info = {'node_ids': hostname_list} return job_info - + #def get_info_on_reserved_nodes(self, job_info, node_list_name): #""" #..warning:unused SA 23/05/13 #""" - ##Get the list of the testbed nodes records and make a + ##Get the list of the testbed nodes records and make a ##dictionnary keyed on the hostname out of it - #node_list_dict = self.GetNodes() + #node_list_dict = self.GetNodes() ##node_hostname_list = [] - #node_hostname_list = [node['hostname'] for node in node_list_dict] + #node_hostname_list = [node['hostname'] for node in node_list_dict] ##for node in node_list_dict: ##node_hostname_list.append(node['hostname']) #node_dict = dict(zip(node_hostname_list, node_list_dict)) #try : #reserved_node_hostname_list = [] #for index in range(len(job_info[node_list_name])): - ##job_info[node_list_name][k] = + ##job_info[node_list_name][k] = #reserved_node_hostname_list[index] = \ #node_dict[job_info[node_list_name][index]]['hostname'] - - #logger.debug("SLABDRIVER \t get_info_on_reserved_nodes \ + + #logger.debug("IOTLAB_API \t get_info_on_reserved_nodes \ #reserved_node_hostname_list %s" \ #%(reserved_node_hostname_list)) #except KeyError: - #logger.error("SLABDRIVER \t get_info_on_reserved_nodes KEYERROR " ) - - #return reserved_node_hostname_list - + #logger.error("IOTLAB_API \t get_info_on_reserved_nodes KEYERROR " ) + + #return reserved_node_hostname_list + def GetNodesCurrentlyInUse(self): - """Returns a list of all the nodes already involved in an oar running + """Returns a list of all the nodes already involved in an oar running job. - :rtype: list of nodes hostnames. + :rtype: list of nodes hostnames. """ - return self.oar.parser.SendRequest("GET_running_jobs") - - def __get_hostnames_from_oar_node_ids(self, resource_id_list ): + return self.oar.parser.SendRequest("GET_running_jobs") + + def __get_hostnames_from_oar_node_ids(self, oar_id_node_dict, + resource_id_list ): """Get the hostnames of the nodes from their OAR identifiers. Get the list of nodes dict using GetNodes and find the hostname associated with the identifier. + :param oar_id_node_dict: full node dictionary list keyed by oar node id :param resource_id_list: list of nodes identifiers :returns: list of node hostnames. """ - full_nodes_dict_list = self.GetNodes() - #Put the full node list into a dictionary keyed by oar node id - oar_id_node_dict = {} - for node in full_nodes_dict_list: - oar_id_node_dict[node['oar_id']] = node - - hostname_list = [] + + hostname_list = [] for resource_id in resource_id_list: #Because jobs requested "asap" do not have defined resources if resource_id is not "Undefined": hostname_list.append(\ oar_id_node_dict[resource_id]['hostname']) - + #hostname_list.append(oar_id_node_dict[resource_id]['hostname']) - return hostname_list - - def GetReservedNodes(self, username = None): + return hostname_list + + def GetReservedNodes(self, username=None): """ Get list of leases. Get the leases for the username if specified, otherwise get all the leases. Finds the nodes hostnames for each OAR node identifier. :param username: user's LDAP login :type username: string - :return: list of reservations dict + :returns: list of reservations dict :rtype: dict list """ - + #Get the nodes in use and the reserved nodes reservation_dict_list = \ self.oar.parser.SendRequest("GET_reserved_nodes", \ username = username) - - + + # Get the full node dict list once for all + # so that we can get the hostnames given their oar node id afterwards + # when the reservations are checked. + full_nodes_dict_list = self.GetNodes() + #Put the full node list into a dictionary keyed by oar node id + oar_id_node_dict = {} + for node in full_nodes_dict_list: + oar_id_node_dict[node['oar_id']] = node + for resa in reservation_dict_list: logger.debug ("GetReservedNodes resa %s"%(resa)) #dict list of hostnames and their site resa['reserved_nodes'] = \ - self.__get_hostnames_from_oar_node_ids(resa['resource_ids']) - + self.__get_hostnames_from_oar_node_ids(oar_id_node_dict, + resa['resource_ids']) + #del resa['resource_ids'] return reservation_dict_list - - def GetNodes(self, node_filter_dict = None, return_fields_list = None): + + def GetNodes(self, node_filter_dict=None, return_fields_list=None): """ - - Make a list of senslab nodes and their properties from information - given by OAR. Search for specific nodes if some filters are specified. - Nodes properties returned if no return_fields_list given: - 'hrn','archi','mobile','hostname','site','boot_state','node_id', - 'radio','posx','posy','oar_id','posz'. - - :param node_filter_dict: dictionnary of lists with node properties + + Make a list of iotlab nodes and their properties from information + given by OAR. Search for specific nodes if some filters are + specified. Nodes properties returned if no return_fields_list given: + 'hrn','archi','mobile','hostname','site','boot_state','node_id', + 'radio','posx','posy','oar_id','posz'. + + :param node_filter_dict: dictionnary of lists with node properties. For + instance, if you want to look for a specific node with its hrn, + the node_filter_dict should be {'hrn': [hrn_of_the_node]} :type node_filter_dict: dict - :param return_fields_list: list of specific fields the user wants to be - returned. + :param return_fields_list: list of specific fields the user wants to be + returned. :type return_fields_list: list - :return: list of dictionaries with node properties + :returns: list of dictionaries with node properties :rtype: list - + """ node_dict_by_id = self.oar.parser.SendRequest("GET_resources_full") node_dict_list = node_dict_by_id.values() - logger.debug (" SLABDRIVER GetNodes node_filter_dict %s \ - return_fields_list %s "%(node_filter_dict, return_fields_list)) + logger.debug (" IOTLAB_API GetNodes node_filter_dict %s \ + return_fields_list %s " % (node_filter_dict, return_fields_list)) #No filtering needed return the list directly if not (node_filter_dict or return_fields_list): return node_dict_list - + return_node_list = [] if node_filter_dict: for filter_key in node_filter_dict: try: - #Filter the node_dict_list by each value contained in the + #Filter the node_dict_list by each value contained in the #list node_filter_dict[filter_key] for value in node_filter_dict[filter_key]: for node in node_dict_list: if node[filter_key] == value: - if return_fields_list : + if return_fields_list: tmp = {} for k in return_fields_list: - tmp[k] = node[k] + tmp[k] = node[k] return_node_list.append(tmp) else: return_node_list.append(node) @@ -383,47 +403,67 @@ class SlabTestbedAPI(): return return_node_list - - - + + + @staticmethod def AddSlice(slice_record, user_record): - """Add slice to the local senslab sfa tables if the slice comes - from a federated site and is not yet in the senslab sfa DB, - although the user has already a LDAP login. - Called by verify_slice during lease/sliver creation. + """ + + Add slice to the local iotlab sfa tables if the slice comes + from a federated site and is not yet in the iotlab sfa DB, + although the user has already a LDAP login. + Called by verify_slice during lease/sliver creation. + :param slice_record: record of slice, must contain hrn, gid, slice_id - and authority of the slice. + and authority of the slice. :type slice_record: dictionary :param user_record: record of the user :type user_record: RegUser + """ - - sfa_record = RegSlice(hrn=slice_record['hrn'], - gid=slice_record['gid'], - pointer=slice_record['slice_id'], - authority=slice_record['authority']) - - logger.debug("SLABDRIVER.PY AddSlice sfa_record %s user_record %s" \ - %(sfa_record, user_record)) + + sfa_record = RegSlice(hrn=slice_record['hrn'], + gid=slice_record['gid'], + pointer=slice_record['slice_id'], + authority=slice_record['authority']) + logger.debug("IOTLAB_API.PY AddSlice sfa_record %s user_record %s" + % (sfa_record, user_record)) sfa_record.just_created() dbsession.add(sfa_record) - dbsession.commit() + dbsession.commit() #Update the reg-researcher dependance table - sfa_record.reg_researchers = [user_record] - dbsession.commit() - + sfa_record.reg_researchers = [user_record] + dbsession.commit() + return - - def GetSites(self, site_filter_name_list = None, return_fields_list = None): + + def GetSites(self, site_filter_name_list=None, return_fields_list=None): + """Returns the list of Iotlab's sites with the associated nodes and + the sites' properties as dictionaries. + + Site properties: + ['address_ids', 'slice_ids', 'name', 'node_ids', 'url', 'person_ids', + 'site_tag_ids', 'enabled', 'site', 'longitude', 'pcu_ids', + 'max_slivers', 'max_slices', 'ext_consortium_id', 'date_created', + 'latitude', 'is_public', 'peer_site_id', 'peer_id', 'abbreviated_name'] + Uses the OAR request GET_sites to find the Iotlab's sites. + + :param site_filter_name_list: used to specify specific sites + :param return_fields_list: field that has to be returned + :type site_filter_name_list: list + :type return_fields_list: list + + + """ site_dict = self.oar.parser.SendRequest("GET_sites") #site_dict : dict where the key is the sit ename return_site_list = [] - if not ( site_filter_name_list or return_fields_list): + if not (site_filter_name_list or return_fields_list): return_site_list = site_dict.values() return return_site_list - + for site_filter_name in site_filter_name_list: if site_filter_name in site_dict: if return_fields_list: @@ -432,45 +472,47 @@ class SlabTestbedAPI(): try: tmp[field] = site_dict[site_filter_name][field] except KeyError: - logger.error("GetSites KeyError %s "%(field)) + logger.error("GetSites KeyError %s " % (field)) return None return_site_list.append(tmp) else: - return_site_list.append( site_dict[site_filter_name]) - + return_site_list.append(site_dict[site_filter_name]) return return_site_list - - - - #TODO : Check rights to delete person + #TODO : Check rights to delete person def DeletePerson(self, person_record): - """ Disable an existing account in senslab LDAP. - Users and techs can only delete themselves. PIs can only - delete themselves and other non-PIs at their sites. - ins can delete anyone. + """Disable an existing account in iotlab LDAP. + + Users and techs can only delete themselves. PIs can only + delete themselves and other non-PIs at their sites. + ins can delete anyone. + :param person_record: user's record :type person_record: dict - :return: True if successful, False otherwise. + :returns: True if successful, False otherwise. :rtype: boolean - + + .. todo:: CHECK THAT ONLY THE USER OR ADMIN CAN DEL HIMSELF. """ - #Disable user account in senslab LDAP + #Disable user account in iotlab LDAP ret = self.ldap.LdapMarkUserAsDeleted(person_record) - logger.warning("SLABDRIVER DeletePerson %s " %(person_record)) + logger.warning("IOTLAB_API DeletePerson %s " % (person_record)) return ret['bool'] - - + def DeleteSlice(self, slice_record): - """ Deletes the specified slice and kills the jobs associated with - the slice if any, using DeleteSliceFromNodes. - - :return: True if all the jobs in the slice have been deleted, - or the list of jobs that could not be deleted otherwise. - :rtype: list or boolean - + """Deletes the specified slice and kills the jobs associated with + the slice if any, using DeleteSliceFromNodes. + + :param slice_record: record of the slice, must contain oar_job_id, user + :type slice_record: dict + :returns: True if all the jobs in the slice have been deleted, + or the list of jobs that could not be deleted otherwise. + :rtype: list or boolean + + .. seealso:: DeleteSliceFromNodes + """ ret = self.DeleteSliceFromNodes(slice_record) delete_failed = None @@ -479,31 +521,31 @@ class SlabTestbedAPI(): if delete_failed is None: delete_failed = [] delete_failed.append(job_id) - - logger.info("SLABDRIVER DeleteSlice %s answer %s"%(slice_record, \ + + logger.info("IOTLAB_API DeleteSlice %s answer %s"%(slice_record, \ delete_failed)) return delete_failed or True - + @staticmethod def __add_person_to_db(user_dict): """ - Add a federated user straight to db when the user issues a lease - request with senslab nodes and that he has not registered with senslab + Add a federated user straight to db when the user issues a lease + request with iotlab nodes and that he has not registered with iotlab yet (that is he does not have a LDAP entry yet). - Uses parts of the routines in SlabImport when importing user from LDAP. + Uses parts of the routines in IotlabImport when importing user from LDAP. Called by AddPerson, right after LdapAddUser. :param user_dict: Must contain email, hrn and pkey to get a GID and be added to the SFA db. :type user_dict: dict - + """ check_if_exists = \ dbsession.query(RegUser).filter_by(email = user_dict['email']).first() #user doesn't exists if not check_if_exists: logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \ - " %(user_dict)) - hrn = user_dict['hrn'] + " %(user_dict)) + hrn = user_dict['hrn'] person_urn = hrn_to_urn(hrn, 'user') pubkey = user_dict['pkey'] try: @@ -513,18 +555,18 @@ class SlabTestbedAPI(): logger.warn('__add_person_to_db: unable to convert public \ key for %s' %(hrn )) pkey = Keypair(create=True) - - + + if pubkey is not None and pkey is not None : hierarchy = Hierarchy() person_gid = hierarchy.create_gid(person_urn, create_uuid(), \ pkey) if user_dict['email']: logger.debug("__add_person_to_db \r\n \r\n \ - SLAB IMPORTER PERSON EMAIL OK email %s "\ + IOTLAB IMPORTER PERSON EMAIL OK email %s "\ %(user_dict['email'])) person_gid.set_email(user_dict['email']) - + user_record = RegUser(hrn=hrn , pointer= '-1', \ authority=get_authority(hrn), \ email=user_dict['email'], gid = person_gid) @@ -532,88 +574,102 @@ class SlabTestbedAPI(): user_record.just_created() dbsession.add (user_record) dbsession.commit() - return - + return + def AddPerson(self, record): - """Adds a new account. Any fields specified in records are used, - otherwise defaults are used. Creates an appropriate login by calling - LdapAddUser. + """ + + Adds a new account. Any fields specified in records are used, + otherwise defaults are used. Creates an appropriate login by calling + LdapAddUser. + :param record: dictionary with the sfa user's properties. - :return: The uid of the added person if sucessful, otherwise returns - the error message from LDAP. - :rtype: interger or string + :returns: a dicitonary with the status. If successful, the dictionary + boolean is set to True and there is a 'uid' key with the new login + added to LDAP, otherwise the bool is set to False and a key + 'message' is in the dictionary, with the error message. + :rtype: dict + """ ret = self.ldap.LdapAddUser(record) - + if ret['bool'] is True: record['hrn'] = self.root_auth + '.' + ret['uid'] - logger.debug("SLABDRIVER AddPerson return code %s record %s \r\n "\ - %(ret, record)) + logger.debug("IOTLAB_API AddPerson return code %s record %s " + % (ret, record)) self.__add_person_to_db(record) - return ret['uid'] - else: - return ret['message'] - - - + return ret + + + + + #TODO AddPersonKey 04/07/2012 SA def AddPersonKey(self, person_uid, old_attributes_dict, new_key_dict): """Adds a new key to the specified account. Adds the key to the - senslab ldap, provided that the person_uid is valid. + iotlab ldap, provided that the person_uid is valid. + Non-admins can only modify their own keys. - - :param person_uid: user's senslab login in LDAP - :param old_attributes_dict: dict with the user's old sshPublicKey - :param new_key_dict:dict with the user's new sshPublicKey + + :param person_uid: user's iotlab login in LDAP + :param old_attributes_dict: dict with the user's old sshPublicKey + :param new_key_dict: dict with the user's new sshPublicKey :type person_uid: string - + :rtype: Boolean - :return: True if the key has been modified, False otherwise. - + :returns: True if the key has been modified, False otherwise. + """ ret = self.ldap.LdapModify(person_uid, old_attributes_dict, \ new_key_dict) - logger.warning("SLABDRIVER AddPersonKey EMPTY - DO NOTHING \r\n ") + logger.warning("IOTLAB_API AddPersonKey EMPTY - DO NOTHING \r\n ") return ret['bool'] - - def DeleteLeases(self, leases_id_list, slice_hrn ): + + def DeleteLeases(self, leases_id_list, slice_hrn): """ + Deletes several leases, based on their job ids and the slice - they are associated with. Uses DeleteJobs to delete the jobs - on OAR. Note that one slice can contain multiple jobs, and in this case - all the jobs in the leases_id_list MUST belong to ONE slice, - since there is only one slice hrn provided here. + they are associated with. Uses DeleteJobs to delete the jobs + on OAR. Note that one slice can contain multiple jobs, and in this + case all the jobs in the leases_id_list MUST belong to ONE slice, + since there is only one slice hrn provided here. + :param leases_id_list: list of job ids that belong to the slice whose - slice hrn is provided. - :param slice_hrn: the slice hrn . - ..warning: Does not have a return value since there was no easy - way to handle failure when dealing with multiple job delete. Plus, - there was no easy way to report it to the user. - """ - logger.debug("SLABDRIVER DeleteLeases leases_id_list %s slice_hrn %s \ + slice hrn is provided. + :param slice_hrn: the slice hrn. + :type slice_hrn: string + + .. warning:: Does not have a return value since there was no easy + way to handle failure when dealing with multiple job delete. Plus, + there was no easy way to report it to the user. + + """ + logger.debug("IOTLAB_API DeleteLeases leases_id_list %s slice_hrn %s \ \r\n " %(leases_id_list, slice_hrn)) for job_id in leases_id_list: self.DeleteJobs(job_id, slice_hrn) - return + return @staticmethod def _process_walltime(duration): """ Calculates the walltime in seconds from the duration in H:M:S specified in the RSpec. - + """ if duration: - # Fixing the walltime by adding a few delays. + # Fixing the walltime by adding a few delays. # First put the walltime in seconds oarAdditionalDelay = 20; # additional delay for /bin/sleep command to # take in account prologue and epilogue scripts execution # int walltimeAdditionalDelay = 240; additional delay + #for prologue/epilogue execution = $SERVER_PROLOGUE_EPILOGUE_TIMEOUT + #in oar.conf # Put the duration in seconds first - desired_walltime = duration * 60 - + #desired_walltime = duration * 60 + desired_walltime = duration total_walltime = desired_walltime + 240 #+4 min Update SA 23/10/12 sleep_walltime = desired_walltime # 0 sec added Update SA 23/10/12 walltime = [] @@ -629,30 +685,30 @@ class SlabTestbedAPI(): else: logger.log_exc(" __process_walltime duration null") - - return walltime, sleep_walltime - - @staticmethod + + return walltime, sleep_walltime + + @staticmethod def _create_job_structure_request_for_OAR(lease_dict): """ Creates the structure needed for a correct POST on OAR. Makes the timestamp transformation into the appropriate format. - Sends the POST request to create the job with the resources in + Sends the POST request to create the job with the resources in added_nodes. - + """ nodeid_list = [] reqdict = {} - - reqdict['workdir'] = '/tmp' - reqdict['resource'] = "{network_address in (" - for node in lease_dict['added_nodes']: + reqdict['workdir'] = '/tmp' + reqdict['resource'] = "{network_address in (" + + for node in lease_dict['added_nodes']: logger.debug("\r\n \r\n OARrestapi \t \ __create_job_structure_request_for_OAR node %s" %(node)) - # Get the ID of the node + # Get the ID of the node nodeid = node reqdict['resource'] += "'" + nodeid + "', " nodeid_list.append(nodeid) @@ -663,7 +719,7 @@ class SlabTestbedAPI(): walltime, sleep_walltime = \ - SlabTestbedAPI._process_walltime(\ + IotlabTestbedAPI._process_walltime(\ int(lease_dict['lease_duration'])) @@ -672,7 +728,7 @@ class SlabTestbedAPI(): reqdict['script_path'] = "/bin/sleep " + str(sleep_walltime) #In case of a scheduled experiment (not immediate) - #To run an XP immediately, don't specify date and time in RSpec + #To run an XP immediately, don't specify date and time in RSpec #They will be set to None. if lease_dict['lease_start_time'] is not '0': #Readable time accepted by OAR @@ -680,28 +736,28 @@ class SlabTestbedAPI(): int(lease_dict['lease_start_time'])).\ strftime(lease_dict['time_format']) reqdict['reservation'] = start_time - #If there is not start time, Immediate XP. No need to add special + #If there is not start time, Immediate XP. No need to add special # OAR parameters - reqdict['type'] = "deploy" + reqdict['type'] = "deploy" reqdict['directory'] = "" reqdict['name'] = "SFA_" + lease_dict['slice_user'] return reqdict - - + + def LaunchExperimentOnOAR(self, added_nodes, slice_name, \ lease_start_time, lease_duration, slice_user=None): - + """ - Create a job request structure based on the information provided - and post the job on OAR. + Create a job request structure based on the information provided + and post the job on OAR. :param added_nodes: list of nodes that belong to the described lease. :param slice_name: the slice hrn associated to the lease. :param lease_start_time: timestamp of the lease startting time. :param lease_duration: lease durationin minutes - + """ lease_dict = {} lease_dict['lease_start_time'] = lease_start_time @@ -712,56 +768,56 @@ class SlabTestbedAPI(): lease_dict['grain'] = self.GetLeaseGranularity() lease_dict['time_format'] = self.time_format - - logger.debug("SLABDRIVER.PY \tLaunchExperimentOnOAR slice_user %s\ - \r\n " %(slice_user)) + + logger.debug("IOTLAB_API.PY \tLaunchExperimentOnOAR slice_user %s\ + \r\n " %(slice_user)) #Create the request for OAR reqdict = self._create_job_structure_request_for_OAR(lease_dict) - # first step : start the OAR job and update the job - logger.debug("SLABDRIVER.PY \tLaunchExperimentOnOAR reqdict %s\ - \r\n " %(reqdict)) - + # first step : start the OAR job and update the job + logger.debug("IOTLAB_API.PY \tLaunchExperimentOnOAR reqdict %s\ + \r\n " %(reqdict)) + answer = self.oar.POSTRequestToOARRestAPI('POST_job', \ reqdict, slice_user) - logger.debug("SLABDRIVER \tLaunchExperimentOnOAR jobid %s " %(answer)) - try: + logger.debug("IOTLAB_API \tLaunchExperimentOnOAR jobid %s " %(answer)) + try: jobid = answer['id'] except KeyError: - logger.log_exc("SLABDRIVER \tLaunchExperimentOnOAR \ + logger.log_exc("IOTLAB_API \tLaunchExperimentOnOAR \ Impossible to create job %s " %(answer)) return None - - - + + + if jobid : - logger.debug("SLABDRIVER \tLaunchExperimentOnOAR jobid %s \ + logger.debug("IOTLAB_API \tLaunchExperimentOnOAR jobid %s \ added_nodes %s slice_user %s" %(jobid, added_nodes, \ slice_user)) - - + + return jobid - - - def AddLeases(self, hostname_list, slice_record, \ - lease_start_time, lease_duration): - + + + def AddLeases(self, hostname_list, slice_record, + lease_start_time, lease_duration): + """Creates a job in OAR corresponding to the information provided - as parameters. Adds the job id and the slice hrn in the senslab + as parameters. Adds the job id and the slice hrn in the iotlab database so that we are able to know which slice has which nodes. - - :param hostname_list: list of nodes' OAR hostnames. + + :param hostname_list: list of nodes' OAR hostnames. :param slice_record: sfa slice record, must contain login and hrn. :param lease_start_time: starting time , unix timestamp format :param lease_duration: duration in minutes - + :type hostname_list: list :type slice_record: dict :type lease_start_time: integer :type lease_duration: integer - + """ - logger.debug("SLABDRIVER \r\n \r\n \t AddLeases hostname_list %s \ + logger.debug("IOTLAB_API \r\n \r\n \t AddLeases hostname_list %s \ slice_record %s lease_start_time %s lease_duration %s "\ %( hostname_list, slice_record , lease_start_time, \ lease_duration)) @@ -778,171 +834,224 @@ class SlabTestbedAPI(): strftime(self.time_format) end_time = lease_start_time + lease_duration - - logger.debug("SLABDRIVER \r\n \r\n \t AddLeases TURN ON LOGGING SQL \ + + logger.debug("IOTLAB_API \r\n \r\n \t AddLeases TURN ON LOGGING SQL \ %s %s %s "%(slice_record['hrn'], job_id, end_time)) - - - logger.debug("SLABDRIVER \r\n \r\n \t AddLeases %s %s %s " \ + + + logger.debug("IOTLAB_API \r\n \r\n \t AddLeases %s %s %s " \ %(type(slice_record['hrn']), type(job_id), type(end_time))) - - slab_ex_row = SenslabXP(slice_hrn = slice_record['hrn'], \ - job_id = job_id, end_time= end_time) - - logger.debug("SLABDRIVER \r\n \r\n \t AddLeases slab_ex_row %s" \ - %(slab_ex_row)) - slab_dbsession.add(slab_ex_row) - slab_dbsession.commit() - - logger.debug("SLABDRIVER \t AddLeases hostname_list start_time %s " \ + + iotlab_ex_row = LeaseTableXP(slice_hrn = slice_record['hrn'], experiment_id=job_id, + end_time= end_time) + + logger.debug("IOTLAB_API \r\n \r\n \t AddLeases iotlab_ex_row %s" \ + %(iotlab_ex_row)) + self.iotlab_db.testbed_session.add(iotlab_ex_row) + self.iotlab_db.testbed_session.commit() + + logger.debug("IOTLAB_API \t AddLeases hostname_list start_time %s " \ %(start_time)) - + return - - - #Delete the jobs from job_senslab table + + + #Delete the jobs from job_iotlab table def DeleteSliceFromNodes(self, slice_record): - """ Deletes all the running or scheduled jobs of a given slice - given its record. - :param slice_record: record of the slice + """ + + Deletes all the running or scheduled jobs of a given slice + given its record. + + :param slice_record: record of the slice, must contain oar_job_id, user :type slice_record: dict - - :return: dict of the jobs'deletion status. Success= True, Failure= - False, for each job id. + + :returns: dict of the jobs'deletion status. Success= True, Failure= + False, for each job id. :rtype: dict + """ - logger.debug("SLABDRIVER \t DeleteSliceFromNodese %s " %(slice_record)) - - if isinstance(slice_record['oar_job_id'], list): + logger.debug("IOTLAB_API \t DeleteSliceFromNodes %s " + % (slice_record)) + + if isinstance(slice_record['oar_job_id'], list): oar_bool_answer = {} for job_id in slice_record['oar_job_id']: ret = self.DeleteJobs(job_id, slice_record['user']) - + oar_bool_answer.update(ret) else: - oar_bool_answer = [self.DeleteJobs(slice_record['oar_job_id'], \ - slice_record['user'])] - + oar_bool_answer = [self.DeleteJobs(slice_record['oar_job_id'], + slice_record['user'])] + return oar_bool_answer - - - + + + def GetLeaseGranularity(self): - """ Returns the granularity of an experiment in the Senslab testbed. - OAR uses seconds for experiments duration , the granulaity is also - defined in seconds. + """ Returns the granularity of an experiment in the Iotlab testbed. + OAR uses seconds for experiments duration , the granulaity is also + defined in seconds. Experiments which last less than 10 min (600 sec) are invalid""" return self.grain - - + + + # @staticmethod + # def update_experiments_in_additional_sfa_db( job_oar_list, jobs_psql): + # """ Cleans the iotlab db by deleting expired and cancelled jobs. + # Compares the list of job ids given by OAR with the job ids that + # are already in the database, deletes the jobs that are no longer in + # the OAR job id list. + # :param job_oar_list: list of job ids coming from OAR + # :type job_oar_list: list + # :param job_psql: list of job ids cfrom the database. + # type job_psql: list + # """ + # #Turn the list into a set + # set_jobs_psql = set(jobs_psql) + + # kept_jobs = set(job_oar_list).intersection(set_jobs_psql) + # logger.debug ( "\r\n \t\ update_experiments_in_additional_sfa_db jobs_psql %s \r\n \t \ + # job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs)) + # deleted_jobs = set_jobs_psql.difference(kept_jobs) + # deleted_jobs = list(deleted_jobs) + # if len(deleted_jobs) > 0: + # self.iotlab_db.testbed_session.query(LeaseTableXP).filter(LeaseTableXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') + # self.iotlab_db.testbed_session.commit() + + # return + @staticmethod - def update_jobs_in_slabdb( job_oar_list, jobs_psql): - """ Cleans the slab db by deleting expired and cancelled jobs. - Compares the list of job ids given by OAR with the job ids that - are already in the database, deletes the jobs that are no longer in - the OAR job id list. - :param job_oar_list: list of job ids coming from OAR - :type job_oar_list: list - :param job_psql: list of job ids cfrom the database. - type job_psql: list - """ - #Turn the list into a set - set_jobs_psql = set(jobs_psql) - - kept_jobs = set(job_oar_list).intersection(set_jobs_psql) - logger.debug ( "\r\n \t\ update_jobs_in_slabdb jobs_psql %s \r\n \t \ - job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs)) - deleted_jobs = set_jobs_psql.difference(kept_jobs) - deleted_jobs = list(deleted_jobs) - if len(deleted_jobs) > 0: - slab_dbsession.query(SenslabXP).filter(SenslabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') - slab_dbsession.commit() - - return + def filter_lease_name(reservation_list, filter_value): + filtered_reservation_list = list(reservation_list) + logger.debug("IOTLAB_API \t filter_lease_name reservation_list %s" \ + % (reservation_list)) + for reservation in reservation_list: + if 'slice_hrn' in reservation and \ + reservation['slice_hrn'] != filter_value: + filtered_reservation_list.remove(reservation) + + logger.debug("IOTLAB_API \t filter_lease_name filtered_reservation_list %s" \ + % (filtered_reservation_list)) + return filtered_reservation_list + + @staticmethod + def filter_lease_start_time(reservation_list, filter_value): + filtered_reservation_list = list(reservation_list) + + for reservation in reservation_list: + if 't_from' in reservation and \ + reservation['t_from'] > filter_value: + filtered_reservation_list.remove(reservation) + + return filtered_reservation_list + - - def GetLeases(self, lease_filter_dict=None, login=None): - """ Get the list of leases from OAR with complete information - about which slice owns which jobs and nodes. - Two purposes: - -Fetch all the jobs from OAR (running, waiting..) - complete the reservation information with slice hrn - found in slabxp table. If not available in the table, - assume it is a senslab slice. - -Updates the slab table, deleting jobs when necessary. - :return: reservation_list, list of dictionaries with 'lease_id', - 'reserved_nodes','slice_id', 'state', 'user', 'component_id_list', - 'slice_hrn', 'resource_ids', 't_from', 't_until' + """ + + Get the list of leases from OAR with complete information + about which slice owns which jobs and nodes. + Two purposes: + -Fetch all the jobs from OAR (running, waiting..) + complete the reservation information with slice hrn + found in testbed_xp table. If not available in the table, + assume it is a iotlab slice. + -Updates the iotlab table, deleting jobs when necessary. + + :returns: reservation_list, list of dictionaries with 'lease_id', + 'reserved_nodes','slice_id', 'state', 'user', 'component_id_list', + 'slice_hrn', 'resource_ids', 't_from', 't_until' :rtype: list + """ - + unfiltered_reservation_list = self.GetReservedNodes(login) reservation_list = [] - #Find the slice associated with this user senslab ldap uid - logger.debug(" SLABDRIVER.PY \tGetLeases login %s\ - unfiltered_reservation_list %s " %(login, unfiltered_reservation_list)) + #Find the slice associated with this user iotlab ldap uid + logger.debug(" IOTLAB_API.PY \tGetLeases login %s\ + unfiltered_reservation_list %s " + % (login, unfiltered_reservation_list)) #Create user dict first to avoid looking several times for #the same user in LDAP SA 27/07/12 job_oar_list = [] - - jobs_psql_query = slab_dbsession.query(SenslabXP).all() - jobs_psql_dict = dict([(row.job_id, row.__dict__ ) for row in jobs_psql_query ]) + + jobs_psql_query = self.iotlab_db.testbed_session.query(LeaseTableXP).all() + jobs_psql_dict = dict([(row.experiment_id, row.__dict__) + for row in jobs_psql_query]) #jobs_psql_dict = jobs_psql_dict) - logger.debug("SLABDRIVER \tGetLeases jobs_psql_dict %s"\ - %(jobs_psql_dict)) - jobs_psql_id_list = [ row.job_id for row in jobs_psql_query ] - - - + logger.debug("IOTLAB_API \tGetLeases jobs_psql_dict %s" + % (jobs_psql_dict)) + jobs_psql_id_list = [row.experiment_id for row in jobs_psql_query] + for resa in unfiltered_reservation_list: - logger.debug("SLABDRIVER \tGetLeases USER %s"\ - %(resa['user'])) - #Construct list of jobs (runing, waiting..) in oar - job_oar_list.append(resa['lease_id']) - #If there is information on the job in SLAB DB ] - #(slice used and job id) + logger.debug("IOTLAB_API \tGetLeases USER %s" + % (resa['user'])) + #Construct list of jobs (runing, waiting..) in oar + job_oar_list.append(resa['lease_id']) + #If there is information on the job in IOTLAB DB ] + #(slice used and job id) if resa['lease_id'] in jobs_psql_dict: job_info = jobs_psql_dict[resa['lease_id']] - logger.debug("SLABDRIVER \tGetLeases job_info %s"\ - %(job_info)) + logger.debug("IOTLAB_API \tGetLeases job_info %s" + % (job_info)) resa['slice_hrn'] = job_info['slice_hrn'] resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice') - - #otherwise, assume it is a senslab slice: + + #otherwise, assume it is a iotlab slice: else: - resa['slice_id'] = hrn_to_urn(self.root_auth+'.'+ \ - resa['user'] +"_slice" , 'slice') + resa['slice_id'] = hrn_to_urn(self.root_auth + '.' + + resa['user'] + "_slice", 'slice') resa['slice_hrn'] = Xrn(resa['slice_id']).get_hrn() - resa['component_id_list'] = [] + resa['component_id_list'] = [] #Transform the hostnames into urns (component ids) for node in resa['reserved_nodes']: - - slab_xrn = slab_xrn_object(self.root_auth, node) - resa['component_id_list'].append(slab_xrn.urn) - - if lease_filter_dict: - logger.debug("SLABDRIVER \tGetLeases resa_ %s \ - \r\n leasefilter %s" %(resa, lease_filter_dict)) - - if lease_filter_dict['name'] == resa['slice_hrn']: - reservation_list.append(resa) - + + iotlab_xrn = iotlab_xrn_object(self.root_auth, node) + resa['component_id_list'].append(iotlab_xrn.urn) + + if lease_filter_dict: + logger.debug("IOTLAB_API \tGetLeases \ + \r\n leasefilter %s" % ( lease_filter_dict)) + + filter_dict_functions = { + 'slice_hrn' : IotlabTestbedAPI.filter_lease_name, + 't_from' : IotlabTestbedAPI.filter_lease_start_time + } + reservation_list = list(unfiltered_reservation_list) + for filter_type in lease_filter_dict: + logger.debug("IOTLAB_API \tGetLeases reservation_list %s" \ + % (reservation_list)) + reservation_list = filter_dict_functions[filter_type](\ + reservation_list,lease_filter_dict[filter_type] ) + + # Filter the reservation list with a maximum timespan so that the + # leases and jobs running after this timestamp do not appear + # in the result leases. + # if 'start_time' in : + # if resa['start_time'] < lease_filter_dict['start_time']: + # reservation_list.append(resa) + + + # if 'name' in lease_filter_dict and \ + # lease_filter_dict['name'] == resa['slice_hrn']: + # reservation_list.append(resa) + + if lease_filter_dict is None: reservation_list = unfiltered_reservation_list - - - self.update_jobs_in_slabdb(job_oar_list, jobs_psql_id_list) - - logger.debug(" SLABDRIVER.PY \tGetLeases reservation_list %s"\ - %(reservation_list)) + + self.iotlab_db.update_experiments_in_additional_sfa_db(job_oar_list, jobs_psql_id_list) + + logger.debug(" IOTLAB_API.PY \tGetLeases reservation_list %s" + % (reservation_list)) return reservation_list - - - + + + #TODO FUNCTIONS SECTION 04/07/2012 SA @@ -951,205 +1060,215 @@ class SlabTestbedAPI(): #@staticmethod #def UnBindObjectFromPeer( auth, object_type, object_id, shortname): #""" This method is a hopefully temporary hack to let the sfa correctly - #detach the objects it creates from a remote peer object. This is - #needed so that the sfa federation link can work in parallel with - #RefreshPeer, as RefreshPeer depends on remote objects being correctly + #detach the objects it creates from a remote peer object. This is + #needed so that the sfa federation link can work in parallel with + #RefreshPeer, as RefreshPeer depends on remote objects being correctly #marked. #Parameters: #auth : struct, API authentication structure - #AuthMethod : string, Authentication method to use + #AuthMethod : string, Authentication method to use #object_type : string, Object type, among 'site','person','slice', #'node','key' #object_id : int, object_id - #shortname : string, peer shortname + #shortname : string, peer shortname #FROM PLC DOC - + #""" - #logger.warning("SLABDRIVER \tUnBindObjectFromPeer EMPTY-\ + #logger.warning("IOTLAB_API \tUnBindObjectFromPeer EMPTY-\ #DO NOTHING \r\n ") - #return - - ##TODO Is BindObjectToPeer still necessary ? Currently does nothing + #return + + ##TODO Is BindObjectToPeer still necessary ? Currently does nothing ##04/07/2012 SA #|| Commented out 28/05/13 SA #def BindObjectToPeer(self, auth, object_type, object_id, shortname=None, \ #remote_object_id=None): - #"""This method is a hopefully temporary hack to let the sfa correctly - #attach the objects it creates to a remote peer object. This is needed - #so that the sfa federation link can work in parallel with RefreshPeer, + #"""This method is a hopefully temporary hack to let the sfa correctly + #attach the objects it creates to a remote peer object. This is needed + #so that the sfa federation link can work in parallel with RefreshPeer, #as RefreshPeer depends on remote objects being correctly marked. #Parameters: - #shortname : string, peer shortname - #remote_object_id : int, remote object_id, set to 0 if unknown + #shortname : string, peer shortname + #remote_object_id : int, remote object_id, set to 0 if unknown #FROM PLC API DOC - + #""" - #logger.warning("SLABDRIVER \tBindObjectToPeer EMPTY - DO NOTHING \r\n ") + #logger.warning("IOTLAB_API \tBindObjectToPeer EMPTY - DO NOTHING \r\n ") #return - + ##TODO UpdateSlice 04/07/2012 SA || Commented out 28/05/13 SA - ##Funciton should delete and create another job since oin senslab slice=job - #def UpdateSlice(self, auth, slice_id_or_name, slice_fields=None): - #"""Updates the parameters of an existing slice with the values in + ##Funciton should delete and create another job since oin iotlab slice=job + #def UpdateSlice(self, auth, slice_id_or_name, slice_fields=None): + #"""Updates the parameters of an existing slice with the values in #slice_fields. - #Users may only update slices of which they are members. - #PIs may update any of the slices at their sites, or any slices of + #Users may only update slices of which they are members. + #PIs may update any of the slices at their sites, or any slices of #which they are members. Admins may update any slice. #Only PIs and admins may update max_nodes. Slices cannot be renewed #(by updating the expires parameter) more than 8 weeks into the future. #Returns 1 if successful, faults otherwise. #FROM PLC API DOC - - #""" - #logger.warning("SLABDRIVER UpdateSlice EMPTY - DO NOTHING \r\n ") + + #""" + #logger.warning("IOTLAB_API UpdateSlice EMPTY - DO NOTHING \r\n ") #return - + #Unused SA 30/05/13, we only update the user's key or we delete it. ##TODO UpdatePerson 04/07/2012 SA - #def UpdatePerson(self, slab_hrn, federated_hrn, person_fields=None): - #"""Updates a person. Only the fields specified in person_fields + #def UpdatePerson(self, iotlab_hrn, federated_hrn, person_fields=None): + #"""Updates a person. Only the fields specified in person_fields #are updated, all other fields are left untouched. #Users and techs can only update themselves. PIs can only update #themselves and other non-PIs at their sites. #Returns 1 if successful, faults otherwise. #FROM PLC API DOC - + #""" - ##new_row = FederatedToSenslab(slab_hrn, federated_hrn) - ##slab_dbsession.add(new_row) - ##slab_dbsession.commit() - - #logger.debug("SLABDRIVER UpdatePerson EMPTY - DO NOTHING \r\n ") + ##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn) + ##self.iotlab_db.testbed_session.add(new_row) + ##self.iotlab_db.testbed_session.commit() + + #logger.debug("IOTLAB_API UpdatePerson EMPTY - DO NOTHING \r\n ") #return - + @staticmethod def GetKeys(key_filter=None): """Returns a dict of dict based on the key string. Each dict entry - contains the key id, the ssh key, the user's email and the + contains the key id, the ssh key, the user's email and the user's hrn. - If key_filter is specified and is an array of key identifiers, - only keys matching the filter will be returned. + If key_filter is specified and is an array of key identifiers, + only keys matching the filter will be returned. Admin may query all keys. Non-admins may only query their own keys. FROM PLC API DOC - - :return: dict with ssh key as key and dicts as value. + + :returns: dict with ssh key as key and dicts as value. :rtype: dict """ if key_filter is None: keys = dbsession.query(RegKey).options(joinedload('reg_user')).all() - else : + else: keys = dbsession.query(RegKey).options(joinedload('reg_user')).filter(RegKey.key.in_(key_filter)).all() - + key_dict = {} for key in keys: - key_dict[key.key] = {'key_id': key.key_id, 'key': key.key, \ - 'email': key.reg_user.email, 'hrn':key.reg_user.hrn} - + key_dict[key.key] = {'key_id': key.key_id, 'key': key.key, + 'email': key.reg_user.email, + 'hrn': key.reg_user.hrn} + #ldap_rslt = self.ldap.LdapSearch({'enabled']=True}) #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \ #for user in ldap_rslt) - - logger.debug("SLABDRIVER GetKeys -key_dict %s \r\n " %(key_dict)) + + logger.debug("IOTLAB_API GetKeys -key_dict %s \r\n " % (key_dict)) return key_dict - + #TODO : test def DeleteKey(self, user_record, key_string): - """ Deletes a key in the LDAP entry of the specified user. + """Deletes a key in the LDAP entry of the specified user. + Removes the key_string from the user's key list and updates the LDAP - user's entry with the new key attributes. + user's entry with the new key attributes. + :param key_string: The ssh key to remove :param user_record: User's record :type key_string: string :type user_record: dict - :return: True if sucessful, False if not. + :returns: True if sucessful, False if not. :rtype: Boolean - + """ all_user_keys = user_record['keys'] all_user_keys.remove(key_string) - new_attributes = {'sshPublicKey':all_user_keys} + new_attributes = {'sshPublicKey':all_user_keys} ret = self.ldap.LdapModifyUser(user_record, new_attributes) - logger.debug("SLABDRIVER DeleteKey %s- "%(ret)) + logger.debug("IOTLAB_API DeleteKey %s- " % (ret)) return ret['bool'] - - - - @staticmethod - def _sql_get_slice_info( slice_filter ): + + + + @staticmethod + def _sql_get_slice_info(slice_filter): """ - Get the slice record based on the slice hrn. Fetch the record of the - user associated with the slice by usingjoinedload based on t - he reg_researcher relationship. + Get the slice record based on the slice hrn. Fetch the record of the + user associated with the slice by using joinedload based on the + reg_researcher relationship. + :param slice_filter: the slice hrn we are looking for :type slice_filter: string - :return: the slice record enhanced with the user's information if the - slice was found, None it wasn't. + :returns: the slice record enhanced with the user's information if the + slice was found, None it wasn't. + :rtype: dict or None. """ - #DO NOT USE RegSlice - reg_researchers to get the hrn - #of the user otherwise will mess up the RegRecord in + #DO NOT USE RegSlice - reg_researchers to get the hrn + #of the user otherwise will mess up the RegRecord in #Resolve, don't know why - SA 08/08/2012 - - #Only one entry for one user = one slice in slab_xp table + + #Only one entry for one user = one slice in testbed_xp table #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() - raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn = slice_filter).first() + raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first() #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() - if raw_slicerec: + if raw_slicerec: #load_reg_researcher #raw_slicerec.reg_researchers raw_slicerec = raw_slicerec.__dict__ - logger.debug(" SLABDRIVER \t get_slice_info slice_filter %s \ - raw_slicerec %s"%(slice_filter, raw_slicerec)) + logger.debug(" IOTLAB_API \t _sql_get_slice_info slice_filter %s \ + raw_slicerec %s" % (slice_filter, raw_slicerec)) slicerec = raw_slicerec #only one researcher per slice so take the first one #slicerec['reg_researchers'] = raw_slicerec['reg_researchers'] #del slicerec['reg_researchers']['_sa_instance_state'] return slicerec - - else : + + else: return None - - @staticmethod - def _sql_get_slice_info_from_user(slice_filter ): + + @staticmethod + def _sql_get_slice_info_from_user(slice_filter): """ - Get the slice record based on the user recordid by using a joinedload + Get the slice record based on the user recordid by using a joinedload on the relationship reg_slices_as_researcher. Format the sql record into a dict with the mandatory fields for user and slice. - :return: dict with slice record and user record if the record was found + :returns: dict with slice record and user record if the record was found based on the user's id, None if not.. :rtype:dict or None.. """ #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() - raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id = slice_filter).first() + raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id=slice_filter).first() #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() - #Put it in correct order - user_needed_fields = ['peer_authority', 'hrn', 'last_updated', 'classtype', 'authority', 'gid', 'record_id', 'date_created', 'type', 'email', 'pointer'] - slice_needed_fields = ['peer_authority', 'hrn', 'last_updated', 'classtype', 'authority', 'gid', 'record_id', 'date_created', 'type', 'pointer'] + #Put it in correct order + user_needed_fields = ['peer_authority', 'hrn', 'last_updated', + 'classtype', 'authority', 'gid', 'record_id', + 'date_created', 'type', 'email', 'pointer'] + slice_needed_fields = ['peer_authority', 'hrn', 'last_updated', + 'classtype', 'authority', 'gid', 'record_id', + 'date_created', 'type', 'pointer'] if raw_slicerec: #raw_slicerec.reg_slices_as_researcher raw_slicerec = raw_slicerec.__dict__ slicerec = {} slicerec = \ - dict([(k, raw_slicerec['reg_slices_as_researcher'][0].__dict__[k]) \ - for k in slice_needed_fields]) - slicerec['reg_researchers'] = dict([(k, raw_slicerec[k]) \ - for k in user_needed_fields]) + dict([(k, raw_slicerec[ + 'reg_slices_as_researcher'][0].__dict__[k]) + for k in slice_needed_fields]) + slicerec['reg_researchers'] = dict([(k, raw_slicerec[k]) + for k in user_needed_fields]) #TODO Handle multiple slices for one user SA 10/12/12 #for now only take the first slice record associated to the rec user ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__ #del raw_slicerec['reg_slices_as_researcher'] #slicerec['reg_researchers'] = raw_slicerec ##del slicerec['_sa_instance_state'] - + return slicerec - + else: return None - - def _get_slice_records(self, slice_filter = None, \ - slice_filter_type = None): + + def _get_slice_records(self, slice_filter=None, + slice_filter_type=None): """ Get the slice record depending on the slice filter and its type. :param slice_filter: Can be either the slice hrn or the user's record @@ -1157,240 +1276,256 @@ class SlabTestbedAPI(): :type slice_filter: string :param slice_filter_type: describes the slice filter type used, can be slice_hrn or record_id_user - :type: string - :return: the slice record - :rtype:dict - ..seealso:_sql_get_slice_info_from_user - ..seealso: _sql_get_slice_info + :type: string + :returns: the slice record + :rtype:dict + .. seealso::_sql_get_slice_info_from_user + .. seealso:: _sql_get_slice_info """ - + #Get list of slices based on the slice hrn if slice_filter_type == 'slice_hrn': - + #if get_authority(slice_filter) == self.root_auth: - #login = slice_filter.split(".")[1].split("_")[0] - + #login = slice_filter.split(".")[1].split("_")[0] + slicerec = self._sql_get_slice_info(slice_filter) - + if slicerec is None: - return None - #return login, None - - #Get slice based on user id - if slice_filter_type == 'record_id_user': - + return None + #return login, None + + #Get slice based on user id + if slice_filter_type == 'record_id_user': + slicerec = self._sql_get_slice_info_from_user(slice_filter) - + if slicerec: fixed_slicerec_dict = slicerec - #At this point if there is no login it means + #At this point if there is no login it means #record_id_user filter has been used for filtering #if login is None : - ##If theslice record is from senslab + ##If theslice record is from iotlab #if fixed_slicerec_dict['peer_authority'] is None: - #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0] + #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0] #return login, fixed_slicerec_dict - return fixed_slicerec_dict - - - - def GetSlices(self, slice_filter = None, slice_filter_type = None, \ - login=None): - """ Get the slice records from the slab db and add lease information - if any. - + return fixed_slicerec_dict + else: + return None + + + def GetSlices(self, slice_filter=None, slice_filter_type=None, + login=None): + """Get the slice records from the iotlab db and add lease information + if any. + :param slice_filter: can be the slice hrn or slice record id in the db - depending on the slice_filter_type. - :param slice_filter_type: defines the type of the filtering used, Can be - either 'slice_hrn' or "record_id'. + depending on the slice_filter_type. + :param slice_filter_type: defines the type of the filtering used, Can be + either 'slice_hrn' or "record_id'. :type slice_filter: string :type slice_filter_type: string - :return: a slice dict if slice_filter and slice_filter_type - are specified and a matching entry is found in the db. The result - is put into a list.Or a list of slice dictionnaries if no filters are - specified. - + :returns: a slice dict if slice_filter and slice_filter_type + are specified and a matching entry is found in the db. The result + is put into a list.Or a list of slice dictionnaries if no filters + arespecified. + :rtype: list + """ #login = None authorized_filter_types_list = ['slice_hrn', 'record_id_user'] return_slicerec_dictlist = [] - - #First try to get information on the slice based on the filter provided + + #First try to get information on the slice based on the filter provided if slice_filter_type in authorized_filter_types_list: - fixed_slicerec_dict = \ - self._get_slice_records(slice_filter, slice_filter_type) + fixed_slicerec_dict = self._get_slice_records(slice_filter, + slice_filter_type) + # if the slice was not found in the sfa db + if fixed_slicerec_dict is None: + return return_slicerec_dictlist + slice_hrn = fixed_slicerec_dict['hrn'] - - logger.debug(" SLABDRIVER \tGetSlices login %s \ + + logger.debug(" IOTLAB_API \tGetSlices login %s \ slice record %s slice_filter %s \ - slice_filter_type %s " %(login, \ - fixed_slicerec_dict, slice_filter, \ + slice_filter_type %s " % (login, + fixed_slicerec_dict, slice_filter, slice_filter_type)) - - - #Now we have the slice record fixed_slicerec_dict, get the + + + #Now we have the slice record fixed_slicerec_dict, get the #jobs associated to this slice leases_list = [] - - leases_list = self.GetLeases(login = login) - #If no job is running or no job scheduled - #return only the slice record + + leases_list = self.GetLeases(login=login) + #If no job is running or no job scheduled + #return only the slice record if leases_list == [] and fixed_slicerec_dict: return_slicerec_dictlist.append(fixed_slicerec_dict) - - #If several jobs for one slice , put the slice record into + + # if the jobs running don't belong to the user/slice we are looking + # for + leases_hrn = [lease['slice_hrn'] for lease in leases_list] + if slice_hrn not in leases_hrn: + return_slicerec_dictlist.append(fixed_slicerec_dict) + #If several jobs for one slice , put the slice record into # each lease information dict - - - for lease in leases_list : - slicerec_dict = {} - logger.debug("SLABDRIVER.PY \tGetSlices slice_filter %s \ - \ lease['slice_hrn'] %s" \ - %(slice_filter, lease['slice_hrn'])) - if lease['slice_hrn'] == slice_hrn: - slicerec_dict['slice_hrn'] = lease['slice_hrn'] - slicerec_dict['hrn'] = lease['slice_hrn'] - slicerec_dict['user'] = lease['user'] + for lease in leases_list: + slicerec_dict = {} + logger.debug("IOTLAB_API.PY \tGetSlices slice_filter %s \ + \t lease['slice_hrn'] %s" + % (slice_filter, lease['slice_hrn'])) + if lease['slice_hrn'] == slice_hrn: slicerec_dict['oar_job_id'] = lease['lease_id'] - slicerec_dict.update({'list_node_ids':{'hostname':lease['reserved_nodes']}}) - slicerec_dict.update({'node_ids':lease['reserved_nodes']}) - #Update lease dict with the slice record if fixed_slicerec_dict: fixed_slicerec_dict['oar_job_id'] = [] - fixed_slicerec_dict['oar_job_id'].append(slicerec_dict['oar_job_id']) + fixed_slicerec_dict['oar_job_id'].append( + slicerec_dict['oar_job_id']) slicerec_dict.update(fixed_slicerec_dict) #slicerec_dict.update({'hrn':\ #str(fixed_slicerec_dict['slice_hrn'])}) - - return_slicerec_dictlist.append(slicerec_dict) - logger.debug("SLABDRIVER.PY \tGetSlices \ - OHOHOHOH %s" %(return_slicerec_dictlist )) - - logger.debug("SLABDRIVER.PY \tGetSlices \ + slicerec_dict['slice_hrn'] = lease['slice_hrn'] + slicerec_dict['hrn'] = lease['slice_hrn'] + slicerec_dict['user'] = lease['user'] + slicerec_dict.update( + {'list_node_ids': + {'hostname': lease['reserved_nodes']}}) + slicerec_dict.update({'node_ids': lease['reserved_nodes']}) + + + + return_slicerec_dictlist.append(slicerec_dict) + logger.debug("IOTLAB_API.PY \tGetSlices \ + OHOHOHOH %s" %(return_slicerec_dictlist)) + + logger.debug("IOTLAB_API.PY \tGetSlices \ slicerec_dict %s return_slicerec_dictlist %s \ lease['reserved_nodes'] \ - %s" %(slicerec_dict, return_slicerec_dictlist, \ - lease['reserved_nodes'] )) - - logger.debug("SLABDRIVER.PY \tGetSlices RETURN \ - return_slicerec_dictlist %s" \ - %(return_slicerec_dictlist)) - + %s" % (slicerec_dict, return_slicerec_dictlist, + lease['reserved_nodes'])) + + logger.debug("IOTLAB_API.PY \tGetSlices RETURN \ + return_slicerec_dictlist %s" + % (return_slicerec_dictlist)) + return return_slicerec_dictlist - - + + else: - #Get all slices from the senslab sfa database , - #put them in dict format - #query_slice_list = dbsession.query(RegRecord).all() - query_slice_list = dbsession.query(RegSlice).options(joinedload('reg_researchers')).all() + #Get all slices from the iotlab sfa database , + #put them in dict format + #query_slice_list = dbsession.query(RegRecord).all() + query_slice_list = \ + dbsession.query(RegSlice).options(joinedload('reg_researchers')).all() - for record in query_slice_list: + for record in query_slice_list: tmp = record.__dict__ tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__ #del tmp['reg_researchers']['_sa_instance_state'] return_slicerec_dictlist.append(tmp) #return_slicerec_dictlist.append(record.__dict__) - + #Get all the jobs reserved nodes leases_list = self.GetReservedNodes() - - + for fixed_slicerec_dict in return_slicerec_dictlist: - slicerec_dict = {} - #Check if the slice belongs to a senslab user + slicerec_dict = {} + #Check if the slice belongs to a iotlab user if fixed_slicerec_dict['peer_authority'] is None: - owner = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0] + owner = fixed_slicerec_dict['hrn'].split( + ".")[1].split("_")[0] else: owner = None - for lease in leases_list: + for lease in leases_list: if owner == lease['user']: slicerec_dict['oar_job_id'] = lease['lease_id'] #for reserved_node in lease['reserved_nodes']: - logger.debug("SLABDRIVER.PY \tGetSlices lease %s "\ - %(lease )) - - slicerec_dict.update({'node_ids':lease['reserved_nodes']}) - slicerec_dict.update({'list_node_ids':{'hostname':lease['reserved_nodes']}}) + logger.debug("IOTLAB_API.PY \tGetSlices lease %s " + % (lease)) slicerec_dict.update(fixed_slicerec_dict) + slicerec_dict.update({'node_ids': + lease['reserved_nodes']}) + slicerec_dict.update({'list_node_ids': + {'hostname': + lease['reserved_nodes']}}) + #slicerec_dict.update({'hrn':\ #str(fixed_slicerec_dict['slice_hrn'])}) #return_slicerec_dictlist.append(slicerec_dict) fixed_slicerec_dict.update(slicerec_dict) - - logger.debug("SLABDRIVER.PY \tGetSlices RETURN \ + + logger.debug("IOTLAB_API.PY \tGetSlices RETURN \ return_slicerec_dictlist %s \slice_filter %s " \ %(return_slicerec_dictlist, slice_filter)) return return_slicerec_dictlist - - #Update slice unused, therefore sfa_fields_to_slab_fields unused + + #Update slice unused, therefore sfa_fields_to_iotlab_fields unused #SA 30/05/13 #@staticmethod - #def sfa_fields_to_slab_fields(sfa_type, hrn, record): + #def sfa_fields_to_iotlab_fields(sfa_type, hrn, record): #""" #""" - #slab_record = {} + #iotlab_record = {} ##for field in record: - ## slab_record[field] = record[field] - + ## iotlab_record[field] = record[field] + #if sfa_type == "slice": - ##instantion used in get_slivers ? - #if not "instantiation" in slab_record: - #slab_record["instantiation"] = "senslab-instantiated" - ##slab_record["hrn"] = hrn_to_pl_slicename(hrn) - ##Unused hrn_to_pl_slicename because Slab's hrn already + ##instantion used in get_slivers ? + #if not "instantiation" in iotlab_record: + #iotlab_record["instantiation"] = "iotlab-instantiated" + ##iotlab_record["hrn"] = hrn_to_pl_slicename(hrn) + ##Unused hrn_to_pl_slicename because Iotlab's hrn already ##in the appropriate form SA 23/07/12 - #slab_record["hrn"] = hrn - #logger.debug("SLABDRIVER.PY sfa_fields_to_slab_fields \ - #slab_record %s " %(slab_record['hrn'])) + #iotlab_record["hrn"] = hrn + #logger.debug("IOTLAB_API.PY sfa_fields_to_iotlab_fields \ + #iotlab_record %s " %(iotlab_record['hrn'])) #if "url" in record: - #slab_record["url"] = record["url"] + #iotlab_record["url"] = record["url"] #if "description" in record: - #slab_record["description"] = record["description"] + #iotlab_record["description"] = record["description"] #if "expires" in record: - #slab_record["expires"] = int(record["expires"]) - + #iotlab_record["expires"] = int(record["expires"]) + ##nodes added by OAR only and then imported to SFA ##elif type == "node": - ##if not "hostname" in slab_record: + ##if not "hostname" in iotlab_record: ##if not "hostname" in record: ##raise MissingSfaInfo("hostname") - ##slab_record["hostname"] = record["hostname"] - ##if not "model" in slab_record: - ##slab_record["model"] = "geni" - - ##One authority only + ##iotlab_record["hostname"] = record["hostname"] + ##if not "model" in iotlab_record: + ##iotlab_record["model"] = "geni" + + ##One authority only ##elif type == "authority": - ##slab_record["login_base"] = hrn_to_slab_login_base(hrn) + ##iotlab_record["login_base"] = hrn_to_iotlab_login_base(hrn) + + ##if not "name" in iotlab_record: + ##iotlab_record["name"] = hrn + + ##if not "abbreviated_name" in iotlab_record: + ##iotlab_record["abbreviated_name"] = hrn + + ##if not "enabled" in iotlab_record: + ##iotlab_record["enabled"] = True + + ##if not "is_public" in iotlab_record: + ##iotlab_record["is_public"] = True + + #return iotlab_record + + - ##if not "name" in slab_record: - ##slab_record["name"] = hrn - ##if not "abbreviated_name" in slab_record: - ##slab_record["abbreviated_name"] = hrn - ##if not "enabled" in slab_record: - ##slab_record["enabled"] = True - ##if not "is_public" in slab_record: - ##slab_record["is_public"] = True - #return slab_record - - - - - - - - \ No newline at end of file diff --git a/sfa/iotlab/iotlabdriver.py b/sfa/iotlab/iotlabdriver.py new file mode 100644 index 00000000..84c8a462 --- /dev/null +++ b/sfa/iotlab/iotlabdriver.py @@ -0,0 +1,798 @@ +""" +Implements what a driver should provide for SFA to work. +""" +from sfa.util.faults import SliverDoesNotExist, UnknownSfaType +from sfa.util.sfalogging import logger +from sfa.storage.alchemy import dbsession +from sfa.storage.model import RegRecord + +from sfa.managers.driver import Driver +from sfa.rspecs.version_manager import VersionManager +from sfa.rspecs.rspec import RSpec + +from sfa.util.xrn import Xrn, hrn_to_urn, get_authority + +from sfa.iotlab.iotlabaggregate import IotlabAggregate, iotlab_xrn_to_hostname +from sfa.iotlab.iotlabslices import IotlabSlices + + +from sfa.iotlab.iotlabapi import IotlabTestbedAPI + + +class IotlabDriver(Driver): + """ Iotlab Driver class inherited from Driver generic class. + + Contains methods compliant with the SFA standard and the testbed + infrastructure (calls to LDAP and OAR). + + .. seealso::: Driver class + + """ + def __init__(self, config): + """ + + Sets the iotlab SFA config parameters, + instanciates the testbed api and the iotlab database. + + :param config: iotlab SFA configuration object + :type config: Config object + + """ + Driver.__init__(self, config) + self.config = config + self.iotlab_api = IotlabTestbedAPI(config) + self.cache = None + + def augment_records_with_testbed_info(self, record_list): + """ + + Adds specific testbed info to the records. + + :param record_list: list of sfa dictionaries records + :type record_list: list + :returns: list of records with extended information in each record + :rtype: list + + """ + return self.fill_record_info(record_list) + + def fill_record_info(self, record_list): + """ + + For each SFA record, fill in the iotlab specific and SFA specific + fields in the record. + + :param record_list: list of sfa dictionaries records + :type record_list: list + :returns: list of records with extended information in each record + :rtype: list + + .. warning:: Should not be modifying record_list directly because modi + fication are kept outside the method's scope. Howerver, there is no + other way to do it given the way it's called in registry manager. + + """ + + logger.debug("IOTLABDRIVER \tfill_record_info records %s " + % (record_list)) + if not isinstance(record_list, list): + record_list = [record_list] + + try: + for record in record_list: + + if str(record['type']) == 'node': + # look for node info using GetNodes + # the record is about one node only + filter_dict = {'hrn': [record['hrn']]} + node_info = self.iotlab_api.GetNodes(filter_dict) + # the node_info is about one node only, but it is formatted + # as a list + record.update(node_info[0]) + logger.debug("IOTLABDRIVER.PY \t \ + fill_record_info NODE" % (record)) + + #If the record is a SFA slice record, then add information + #about the user of this slice. This kind of + #information is in the Iotlab's DB. + if str(record['type']) == 'slice': + if 'reg_researchers' in record and isinstance(record + ['reg_researchers'], + list): + record['reg_researchers'] = \ + record['reg_researchers'][0].__dict__ + record.update( + {'PI': [record['reg_researchers']['hrn']], + 'researcher': [record['reg_researchers']['hrn']], + 'name': record['hrn'], + 'oar_job_id': [], + 'node_ids': [], + 'person_ids': [record['reg_researchers'] + ['record_id']], + # For client_helper.py compatibility + 'geni_urn': '', + # For client_helper.py compatibility + 'keys': '', + # For client_helper.py compatibility + 'key_ids': ''}) + + #Get iotlab slice record and oar job id if any. + recslice_list = self.iotlab_api.GetSlices( + slice_filter=str(record['hrn']), + slice_filter_type='slice_hrn') + + logger.debug("IOTLABDRIVER \tfill_record_info \ + TYPE SLICE RECUSER record['hrn'] %s record['oar_job_id']\ + %s " % (record['hrn'], record['oar_job_id'])) + del record['reg_researchers'] + try: + for rec in recslice_list: + logger.debug("IOTLABDRIVER\r\n \t \ + fill_record_info oar_job_id %s " + % (rec['oar_job_id'])) + + record['node_ids'] = [self.iotlab_api.root_auth + + '.' + hostname for hostname + in rec['node_ids']] + except KeyError: + pass + + logger.debug("IOTLABDRIVER.PY \t fill_record_info SLICE \ + recslice_list %s \r\n \t RECORD %s \r\n \ + \r\n" % (recslice_list, record)) + + if str(record['type']) == 'user': + #The record is a SFA user record. + #Get the information about his slice from Iotlab's DB + #and add it to the user record. + recslice_list = self.iotlab_api.GetSlices( + slice_filter=record['record_id'], + slice_filter_type='record_id_user') + + logger.debug("IOTLABDRIVER.PY \t fill_record_info \ + TYPE USER recslice_list %s \r\n \t RECORD %s \r\n" + % (recslice_list, record)) + #Append slice record in records list, + #therefore fetches user and slice info again(one more loop) + #Will update PIs and researcher for the slice + + recuser = recslice_list[0]['reg_researchers'] + logger.debug("IOTLABDRIVER.PY \t fill_record_info USER \ + recuser %s \r\n \r\n" % (recuser)) + recslice = {} + recslice = recslice_list[0] + recslice.update( + {'PI': [recuser['hrn']], + 'researcher': [recuser['hrn']], + 'name': record['hrn'], + 'node_ids': [], + 'oar_job_id': [], + 'person_ids': [recuser['record_id']]}) + try: + for rec in recslice_list: + recslice['oar_job_id'].append(rec['oar_job_id']) + except KeyError: + pass + + recslice.update({'type': 'slice', + 'hrn': recslice_list[0]['hrn']}) + + #GetPersons takes [] as filters + user_iotlab = self.iotlab_api.GetPersons([record]) + + record.update(user_iotlab[0]) + #For client_helper.py compatibility + record.update( + {'geni_urn': '', + 'keys': '', + 'key_ids': ''}) + record_list.append(recslice) + + logger.debug("IOTLABDRIVER.PY \t \ + fill_record_info ADDING SLICE\ + INFO TO USER records %s" % (record_list)) + + except TypeError, error: + logger.log_exc("IOTLABDRIVER \t fill_record_info EXCEPTION %s" + % (error)) + + return record_list + + def sliver_status(self, slice_urn, slice_hrn): + """ + Receive a status request for slice named urn/hrn + urn:publicid:IDN+iotlab+nturro_slice hrn iotlab.nturro_slice + shall return a structure as described in + http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus + NT : not sure if we should implement this or not, but used by sface. + + :param slice_urn: slice urn + :type slice_urn: string + :param slice_hrn: slice hrn + :type slice_hrn: string + + """ + + #First get the slice with the slice hrn + slice_list = self.iotlab_api.GetSlices(slice_filter=slice_hrn, + slice_filter_type='slice_hrn') + + if len(slice_list) == 0: + raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn)) + + #Used for fetching the user info witch comes along the slice info + one_slice = slice_list[0] + + #Make a list of all the nodes hostnames in use for this slice + slice_nodes_list = [] + slice_nodes_list = one_slice['node_ids'] + #Get all the corresponding nodes details + nodes_all = self.iotlab_api.GetNodes( + {'hostname': slice_nodes_list}, + ['node_id', 'hostname', 'site', 'boot_state']) + nodeall_byhostname = dict([(one_node['hostname'], one_node) + for one_node in nodes_all]) + + for single_slice in slice_list: + #For compatibility + top_level_status = 'empty' + result = {} + result.fromkeys( + ['geni_urn', 'geni_error', 'iotlab_login', 'geni_status', + 'geni_resources'], None) + # result.fromkeys(\ + # ['geni_urn','geni_error', 'pl_login','geni_status', + # 'geni_resources'], None) + # result['pl_login'] = one_slice['reg_researchers'][0].hrn + result['iotlab_login'] = one_slice['user'] + logger.debug("Slabdriver - sliver_status Sliver status \ + urn %s hrn %s single_slice %s \r\n " + % (slice_urn, slice_hrn, single_slice)) + + if 'node_ids' not in single_slice: + #No job in the slice + result['geni_status'] = top_level_status + result['geni_resources'] = [] + return result + + top_level_status = 'ready' + + #A job is running on Iotlab for this slice + # report about the local nodes that are in the slice only + + result['geni_urn'] = slice_urn + + resources = [] + for node_hostname in single_slice['node_ids']: + res = {} + res['iotlab_hostname'] = node_hostname + res['iotlab_boot_state'] = \ + nodeall_byhostname[node_hostname]['boot_state'] + + #res['pl_hostname'] = node['hostname'] + #res['pl_boot_state'] = \ + #nodeall_byhostname[node['hostname']]['boot_state'] + #res['pl_last_contact'] = strftime(self.time_format, \ + #gmtime(float(timestamp))) + sliver_id = Xrn( + slice_urn, type='slice', + id=nodeall_byhostname[node_hostname]['node_id']).urn + + res['geni_urn'] = sliver_id + #node_name = node['hostname'] + if nodeall_byhostname[node_hostname]['boot_state'] == 'Alive': + + res['geni_status'] = 'ready' + else: + res['geni_status'] = 'failed' + top_level_status = 'failed' + + res['geni_error'] = '' + + resources.append(res) + + result['geni_status'] = top_level_status + result['geni_resources'] = resources + logger.debug("IOTLABDRIVER \tsliver_statusresources %s res %s " + % (resources, res)) + return result + + @staticmethod + def get_user_record(hrn): + """ + + Returns the user record based on the hrn from the SFA DB . + + :param hrn: user's hrn + :type hrn: string + :returns: user record from SFA database + :rtype: RegUser + + """ + return dbsession.query(RegRecord).filter_by(hrn=hrn).first() + + def testbed_name(self): + """ + + Returns testbed's name. + :returns: testbed authority name. + :rtype: string + + """ + return self.hrn + + # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory + def aggregate_version(self): + """ + + Returns the testbed's supported rspec advertisement and request + versions. + :returns: rspec versions supported ad a dictionary. + :rtype: dict + + """ + version_manager = VersionManager() + ad_rspec_versions = [] + request_rspec_versions = [] + for rspec_version in version_manager.versions: + if rspec_version.content_type in ['*', 'ad']: + ad_rspec_versions.append(rspec_version.to_dict()) + if rspec_version.content_type in ['*', 'request']: + request_rspec_versions.append(rspec_version.to_dict()) + return { + 'testbed': self.testbed_name(), + 'geni_request_rspec_versions': request_rspec_versions, + 'geni_ad_rspec_versions': ad_rspec_versions} + + def _get_requested_leases_list(self, rspec): + """ + Process leases in rspec depending on the rspec version (format) + type. Find the lease requests in the rspec and creates + a lease request list with the mandatory information ( nodes, + start time and duration) of the valid leases (duration above or + equal to the iotlab experiment minimum duration). + + :param rspec: rspec request received. + :type rspec: RSpec + :returns: list of lease requests found in the rspec + :rtype: list + """ + requested_lease_list = [] + for lease in rspec.version.get_leases(): + single_requested_lease = {} + logger.debug("IOTLABDRIVER.PY \t \ + _get_requested_leases_list lease %s " % (lease)) + + if not lease.get('lease_id'): + if get_authority(lease['component_id']) == \ + self.iotlab_api.root_auth: + single_requested_lease['hostname'] = \ + iotlab_xrn_to_hostname(\ + lease.get('component_id').strip()) + single_requested_lease['start_time'] = \ + lease.get('start_time') + single_requested_lease['duration'] = lease.get('duration') + #Check the experiment's duration is valid before adding + #the lease to the requested leases list + duration_in_seconds = \ + int(single_requested_lease['duration']) + if duration_in_seconds >= self.iotlab_api.GetMinExperimentDurationInGranularity(): + requested_lease_list.append(single_requested_lease) + + return requested_lease_list + + @staticmethod + def _group_leases_by_start_time(requested_lease_list): + """ + Create dict of leases by start_time, regrouping nodes reserved + at the same time, for the same amount of time so as to + define one job on OAR. + + :param requested_lease_list: list of leases + :type requested_lease_list: list + :returns: Dictionary with key = start time, value = list of leases + with the same start time. + :rtype: dictionary + + """ + + requested_job_dict = {} + for lease in requested_lease_list: + + #In case it is an asap experiment start_time is empty + if lease['start_time'] == '': + lease['start_time'] = '0' + + if lease['start_time'] not in requested_job_dict: + if isinstance(lease['hostname'], str): + lease['hostname'] = [lease['hostname']] + + requested_job_dict[lease['start_time']] = lease + + else: + job_lease = requested_job_dict[lease['start_time']] + if lease['duration'] == job_lease['duration']: + job_lease['hostname'].append(lease['hostname']) + + return requested_job_dict + + def _process_requested_jobs(self, rspec): + """ + Turns the requested leases and information into a dictionary + of requested jobs, grouped by starting time. + + :param rspec: RSpec received + :type rspec : RSpec + :rtype: dictionary + + """ + requested_lease_list = self._get_requested_leases_list(rspec) + logger.debug("IOTLABDRIVER _process_requested_jobs \ + requested_lease_list %s" % (requested_lease_list)) + job_dict = self._group_leases_by_start_time(requested_lease_list) + logger.debug("IOTLABDRIVER _process_requested_jobs job_dict\ + %s" % (job_dict)) + + return job_dict + + def create_sliver(self, slice_urn, slice_hrn, creds, rspec_string, + users, options): + """Answer to CreateSliver. + + Creates the leases and slivers for the users from the information + found in the rspec string. + Launch experiment on OAR if the requested leases is valid. Delete + no longer requested leases. + + + :param creds: user's credentials + :type creds: string + :param users: user record list + :type users: list + :param options: + :type options: + + :returns: a valid Rspec for the slice which has just been + modified. + :rtype: RSpec + + + """ + aggregate = IotlabAggregate(self) + + slices = IotlabSlices(self) + peer = slices.get_peer(slice_hrn) + sfa_peer = slices.get_sfa_peer(slice_hrn) + slice_record = None + + if not isinstance(creds, list): + creds = [creds] + + if users: + slice_record = users[0].get('slice_record', {}) + logger.debug("IOTLABDRIVER.PY \t ===============create_sliver \t\ + creds %s \r\n \r\n users %s" + % (creds, users)) + slice_record['user'] = {'keys': users[0]['keys'], + 'email': users[0]['email'], + 'hrn': slice_record['reg-researchers'][0]} + # parse rspec + rspec = RSpec(rspec_string) + logger.debug("IOTLABDRIVER.PY \t create_sliver \trspec.version \ + %s slice_record %s users %s" + % (rspec.version, slice_record, users)) + + # ensure site record exists? + # ensure slice record exists + #Removed options in verify_slice SA 14/08/12 + #Removed peer record in verify_slice SA 18/07/13 + sfa_slice = slices.verify_slice(slice_hrn, slice_record, sfa_peer) + + # ensure person records exists + #verify_persons returns added persons but the return value + #is not used + #Removed peer record and sfa_peer in verify_persons SA 18/07/13 + slices.verify_persons(slice_hrn, sfa_slice, users, options=options) + #requested_attributes returned by rspec.version.get_slice_attributes() + #unused, removed SA 13/08/12 + #rspec.version.get_slice_attributes() + + logger.debug("IOTLABDRIVER.PY create_sliver slice %s " % (sfa_slice)) + + # add/remove slice from nodes + + #requested_slivers = [node.get('component_id') \ + #for node in rspec.version.get_nodes_with_slivers()\ + #if node.get('authority_id') is self.iotlab_api.root_auth] + #l = [ node for node in rspec.version.get_nodes_with_slivers() ] + #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \ + #requested_slivers %s listnodes %s" \ + #%(requested_slivers,l)) + #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12. + #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer) + + requested_job_dict = self._process_requested_jobs(rspec) + + logger.debug("IOTLABDRIVER.PY \tcreate_sliver requested_job_dict %s " + % (requested_job_dict)) + #verify_slice_leases returns the leases , but the return value is unused + #here. Removed SA 13/08/12 + slices.verify_slice_leases(sfa_slice, + requested_job_dict, peer) + + return aggregate.get_rspec(slice_xrn=slice_urn, + login=sfa_slice['login'], + version=rspec.version) + + def delete_sliver(self, slice_urn, slice_hrn, creds, options): + """ + Deletes the lease associated with the slice hrn and the credentials + if the slice belongs to iotlab. Answer to DeleteSliver. + + :param slice_urn: urn of the slice + :param slice_hrn: name of the slice + :param creds: slice credenials + :type slice_urn: string + :type slice_hrn: string + :type creds: ? unused + + :returns: 1 if the slice to delete was not found on iotlab, + True if the deletion was successful, False otherwise otherwise. + + .. note:: Should really be named delete_leases because iotlab does + not have any slivers, but only deals with leases. However, + SFA api only have delete_sliver define so far. SA 13/05/2013 + .. note:: creds are unused, and are not used either in the dummy driver + delete_sliver . + """ + + sfa_slice_list = self.iotlab_api.GetSlices( + slice_filter=slice_hrn, + slice_filter_type='slice_hrn') + + if not sfa_slice_list: + return 1 + + #Delete all leases in the slice + for sfa_slice in sfa_slice_list: + logger.debug("IOTLABDRIVER.PY delete_sliver slice %s" % (sfa_slice)) + slices = IotlabSlices(self) + # determine if this is a peer slice + + peer = slices.get_peer(slice_hrn) + + logger.debug("IOTLABDRIVER.PY delete_sliver peer %s \ + \r\n \t sfa_slice %s " % (peer, sfa_slice)) + try: + self.iotlab_api.DeleteSliceFromNodes(sfa_slice) + return True + except: + return False + + def list_resources (self, slice_urn, slice_hrn, creds, options): + """ + + List resources from the iotlab aggregate and returns a Rspec + advertisement with resources found when slice_urn and slice_hrn are + None (in case of resource discovery). + If a slice hrn and urn are provided, list experiment's slice + nodes in a rspec format. Answer to ListResources. + Caching unused. + + :param slice_urn: urn of the slice + :param slice_hrn: name of the slice + :param creds: slice credenials + :type slice_urn: string + :type slice_hrn: string + :type creds: ? unused + :param options: options used when listing resources (list_leases, info, + geni_available) + :returns: rspec string in xml + :rtype: string + + .. note:: creds are unused + """ + + #cached_requested = options.get('cached', True) + + version_manager = VersionManager() + # get the rspec's return format from options + rspec_version = \ + version_manager.get_version(options.get('geni_rspec_version')) + version_string = "rspec_%s" % (rspec_version) + + #panos adding the info option to the caching key (can be improved) + if options.get('info'): + version_string = version_string + "_" + \ + options.get('info', 'default') + + # Adding the list_leases option to the caching key + if options.get('list_leases'): + version_string = version_string + "_" + \ + options.get('list_leases', 'default') + + # Adding geni_available to caching key + if options.get('geni_available'): + version_string = version_string + "_" + \ + str(options.get('geni_available')) + + # look in cache first + #if cached_requested and self.cache and not slice_hrn: + #rspec = self.cache.get(version_string) + #if rspec: + #logger.debug("IotlabDriver.ListResources: \ + #returning cached advertisement") + #return rspec + + #panos: passing user-defined options + aggregate = IotlabAggregate(self) + + rspec = aggregate.get_rspec(slice_xrn=slice_urn, + version=rspec_version, options=options) + + # cache the result + #if self.cache and not slice_hrn: + #logger.debug("Iotlab.ListResources: stores advertisement in cache") + #self.cache.add(version_string, rspec) + + return rspec + + + def list_slices(self, creds, options): + """Answer to ListSlices. + + List slices belonging to iotlab, returns slice urns list. + No caching used. Options unused but are defined in the SFA method + api prototype. + + :returns: slice urns list + :rtype: list + + .. note:: creds are unused + """ + # look in cache first + #if self.cache: + #slices = self.cache.get('slices') + #if slices: + #logger.debug("PlDriver.list_slices returns from cache") + #return slices + + # get data from db + + slices = self.iotlab_api.GetSlices() + logger.debug("IOTLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" + % (slices)) + slice_hrns = [iotlab_slice['hrn'] for iotlab_slice in slices] + + slice_urns = [hrn_to_urn(slice_hrn, 'slice') + for slice_hrn in slice_hrns] + + # cache the result + #if self.cache: + #logger.debug ("IotlabDriver.list_slices stores value in cache") + #self.cache.add('slices', slice_urns) + + return slice_urns + + + def register(self, sfa_record, hrn, pub_key): + """ + Adding new user, slice, node or site should not be handled + by SFA. + + ..warnings:: should not be used. Different components are in charge of + doing this task. Adding nodes = OAR + Adding users = LDAP Iotlab + Adding slice = Import from LDAP users + Adding site = OAR + + :param sfa_record: record provided by the client of the + Register API call. + :type sfa_record: dict + :param pub_key: public key of the user + :type pub_key: string + + .. note:: DOES NOTHING. Returns -1. + + """ + return -1 + + + def update(self, old_sfa_record, new_sfa_record, hrn, new_key): + """ + No site or node record update allowed in Iotlab. The only modifications + authorized here are key deletion/addition on an existing user and + password change. On an existing user, CAN NOT BE MODIFIED: 'first_name', + 'last_name', 'email'. DOES NOT EXIST IN SENSLAB: 'phone', 'url', 'bio', + 'title', 'accepted_aup'. A slice is bound to its user, so modifying the + user's ssh key should nmodify the slice's GID after an import procedure. + + :param old_sfa_record: what is in the db for this hrn + :param new_sfa_record: what was passed to the update call + :param new_key: the new user's public key + :param hrn: the user's sfa hrn + :type old_sfa_record: dict + :type new_sfa_record: dict + :type new_key: string + :type hrn: string + + TODO: needs review + .. seealso:: update in driver.py. + + """ + pointer = old_sfa_record['pointer'] + old_sfa_record_type = old_sfa_record['type'] + + # new_key implemented for users only + if new_key and old_sfa_record_type not in ['user']: + raise UnknownSfaType(old_sfa_record_type) + + if old_sfa_record_type == "user": + update_fields = {} + all_fields = new_sfa_record + for key in all_fields.keys(): + if key in ['key', 'password']: + update_fields[key] = all_fields[key] + + if new_key: + # must check this key against the previous one if it exists + persons = self.iotlab_api.GetPersons([old_sfa_record]) + person = persons[0] + keys = [person['pkey']] + #Get all the person's keys + keys_dict = self.iotlab_api.GetKeys(keys) + + # Delete all stale keys, meaning the user has only one key + #at a time + #TODO: do we really want to delete all the other keys? + #Is this a problem with the GID generation to have multiple + #keys? SA 30/05/13 + key_exists = False + if key in keys_dict: + key_exists = True + else: + #remove all the other keys + for key in keys_dict: + self.iotlab_api.DeleteKey(person, key) + self.iotlab_api.AddPersonKey( + person, {'sshPublicKey': person['pkey']}, + {'sshPublicKey': new_key}) + return True + + def remove(self, sfa_record): + """ + + Removes users only. Mark the user as disabled in LDAP. The user and his + slice are then deleted from the db by running an import on the registry. + + :param sfa_record: record is the existing sfa record in the db + :type sfa_record: dict + + ..warning::As fas as the slice is concerned, here only the leases are + removed from the slice. The slice is record itself is not removed + from the db. + + TODO: needs review + + TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013, + + TODO: return boolean for the slice part + """ + sfa_record_type = sfa_record['type'] + hrn = sfa_record['hrn'] + if sfa_record_type == 'user': + + #get user from iotlab ldap + person = self.iotlab_api.GetPersons(sfa_record) + #No registering at a given site in Iotlab. + #Once registered to the LDAP, all iotlab sites are + #accesible. + if person: + #Mark account as disabled in ldap + return self.iotlab_api.DeletePerson(sfa_record) + + elif sfa_record_type == 'slice': + if self.iotlab_api.GetSlices(slice_filter=hrn, + slice_filter_type='slice_hrn'): + ret = self.iotlab_api.DeleteSlice(sfa_record) + return True diff --git a/sfa/iotlab/iotlabpostgres.py b/sfa/iotlab/iotlabpostgres.py new file mode 100644 index 00000000..eed8769a --- /dev/null +++ b/sfa/iotlab/iotlabpostgres.py @@ -0,0 +1,273 @@ +""" +File defining classes to handle the table in the iotlab dedicated database. +""" + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +# from sfa.util.config import Config +from sfa.util.sfalogging import logger + +from sqlalchemy import Column, Integer, String +from sqlalchemy import Table, MetaData +from sqlalchemy.ext.declarative import declarative_base + +# from sqlalchemy.dialects import postgresql + +from sqlalchemy.exc import NoSuchTableError + + +#Dict holding the columns names of the table as keys +#and their type, used for creation of the table +slice_table = {'record_id_user': 'integer PRIMARY KEY references X ON DELETE \ + CASCADE ON UPDATE CASCADE', 'oar_job_id': 'integer DEFAULT -1', + 'record_id_slice': 'integer', 'slice_hrn': 'text NOT NULL'} + +#Dict with all the specific iotlab tables +# tablenames_dict = {'lease_table': slice_table} + + +TestbedBase = declarative_base() + + +class LeaseTableXP (TestbedBase): + """ SQL alchemy class to manipulate the rows of the slice_iotlab table in + lease_table database. Handles the records representation and creates the + table if it does not exist yet. + + """ + __tablename__ = 'lease_table' + + slice_hrn = Column(String) + experiment_id = Column(Integer, primary_key=True) + end_time = Column(Integer, nullable=False) + + def __init__(self, slice_hrn=None, experiment_id=None, end_time=None): + """ + Defines a row of the slice_iotlab table + """ + if slice_hrn: + self.slice_hrn = slice_hrn + if experiment_id: + self.experiment_id = experiment_id + if end_time: + self.end_time = end_time + + def __repr__(self): + """Prints the SQLAlchemy record to the format defined + by the function. + """ + result = " 0: + self.testbed_session.query(LeaseTableXP).filter(LeaseTableXP.experiment_id.in_(deleted_experiments)).delete(synchronize_session='fetch') + self.testbed_session.commit() + return + + def __init__(self, config, debug=False): + self.sl_base = TestbedBase + + # Check whether we already have an instance + if TestbedAdditionalSfaDB._connection_singleton is None: + TestbedAdditionalSfaDB._connection_singleton = \ + TestbedAdditionalSfaDB.Singleton(config, debug) + + # Store instance reference as the only member in the handle + self._EventHandler_singleton = \ + TestbedAdditionalSfaDB._connection_singleton + + def __getattr__(self, aAttr): + """ + Delegate access to implementation. + + :param aAttr: Attribute wanted. + :returns: Attribute + """ + return getattr(self._connection_singleton, aAttr) + + + + # def __setattr__(self, aAttr, aValue): + # """Delegate access to implementation. + + # :param attr: Attribute wanted. + # :param value: Vaule to be set. + # :return: Result of operation. + # """ + # return setattr(self._connection_singleton, aAttr, aValue) + + def exists(self, tablename): + """ + Checks if the table specified as tablename exists. + :param tablename: name of the table in the db that has to be checked. + :type tablename: string + :returns: True if the table exists, False otherwise. + :rtype: bool + + """ + metadata = MetaData(bind=self.testbed_engine) + try: + table = Table(tablename, metadata, autoload=True) + return True + + except NoSuchTableError: + logger.log_exc("SLABPOSTGRES tablename %s does not exist" + % (tablename)) + return False + + def createtable(self): + """ + Creates all the table sof the engine. + Uses the global dictionnary holding the tablenames and the table schema. + + """ + + logger.debug("IOTLABPOSTGRES createtable \ + TestbedBase.metadata.sorted_tables %s \r\n engine %s" + % (TestbedBase.metadata.sorted_tables, self.testbed_engine)) + TestbedBase.metadata.create_all(self.testbed_engine) + return diff --git a/sfa/iotlab/iotlabslices.py b/sfa/iotlab/iotlabslices.py new file mode 100644 index 00000000..10688405 --- /dev/null +++ b/sfa/iotlab/iotlabslices.py @@ -0,0 +1,587 @@ +""" +This file defines the IotlabSlices class by which all the slice checkings +upon lease creation are done. +""" +from sfa.util.xrn import get_authority, urn_to_hrn +from sfa.util.sfalogging import logger + +MAXINT = 2L**31-1 + + +class IotlabSlices: + """ + This class is responsible for checking the slice when creating a + lease or a sliver. Those checks include verifying that the user is valid, + that the slice is known from the testbed or from our peers, that the list + of nodes involved has not changed (in this case the lease is modified + accordingly). + """ + rspec_to_slice_tag = {'max_rate': 'net_max_rate'} + + def __init__(self, driver): + """ + Get the reference to the driver here. + """ + self.driver = driver + + def get_peer(self, xrn): + """ + Finds the authority of a resource based on its xrn. + If the authority is Iotlab (local) return None, + Otherwise, look up in the DB if Iotlab is federated with this site + authority and returns its DB record if it is the case. + + :param xrn: resource's xrn + :type xrn: string + :returns: peer record + :rtype: dict + + """ + hrn, hrn_type = urn_to_hrn(xrn) + #Does this slice belong to a local site or a peer iotlab site? + peer = None + + # get this slice's authority (site) + slice_authority = get_authority(hrn) + #Iotlab stuff + #This slice belongs to the current site + if slice_authority == self.driver.iotlab_api.root_auth: + site_authority = slice_authority + return None + + site_authority = get_authority(slice_authority).lower() + # get this site's authority (sfa root authority or sub authority) + + logger.debug("IOTLABSLICES \t get_peer slice_authority %s \ + site_authority %s hrn %s" + % (slice_authority, site_authority, hrn)) + + # check if we are already peered with this site_authority + #if so find the peer record + peers = self.driver.iotlab_api.GetPeers(peer_filter=site_authority) + for peer_record in peers: + if site_authority == peer_record.hrn: + peer = peer_record + logger.debug(" IOTLABSLICES \tget_peer peer %s " % (peer)) + return peer + + def get_sfa_peer(self, xrn): + """Returns the authority name for the xrn or None if the local site + is the authority. + + :param xrn: the xrn of the resource we are looking the authority for. + :type xrn: string + :returns: the resources's authority name. + :rtype: string + + """ + hrn, hrn_type = urn_to_hrn(xrn) + + # return the authority for this hrn or None if we are the authority + sfa_peer = None + slice_authority = get_authority(hrn) + site_authority = get_authority(slice_authority) + + if site_authority != self.driver.hrn: + sfa_peer = site_authority + + return sfa_peer + + def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer): + """ + Compare requested leases with the leases already scheduled/ + running in OAR. If necessary, delete and recreate modified leases, + and delete no longer requested ones. + + :param sfa_slice: sfa slice record + :param requested_jobs_dict: dictionary of requested leases + :param peer: sfa peer record + + :type sfa_slice: dict + :type requested_jobs_dict: dict + :type peer: dict + :returns: leases list of dictionary + :rtype: list + + """ + + logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s " + % (sfa_slice)) + #First get the list of current leases from OAR + leases = self.driver.iotlab_api.GetLeases({'slice_hrn': sfa_slice['hrn']}) + logger.debug("IOTLABSLICES verify_slice_leases requested_jobs_dict %s \ + leases %s " % (requested_jobs_dict, leases)) + + current_nodes_reserved_by_start_time = {} + requested_nodes_by_start_time = {} + leases_by_start_time = {} + reschedule_jobs_dict = {} + + #Create reduced dictionary with key start_time and value + # the list of nodes + #-for the leases already registered by OAR first + # then for the new leases requested by the user + + #Leases already scheduled/running in OAR + for lease in leases: + current_nodes_reserved_by_start_time[lease['t_from']] = \ + lease['reserved_nodes'] + leases_by_start_time[lease['t_from']] = lease + + #First remove job whose duration is too short + for job in requested_jobs_dict.values(): + job['duration'] = \ + str(int(job['duration']) \ + * self.driver.iotlab_api.GetLeaseGranularity()) + if job['duration'] < self.driver.iotlab_api.GetLeaseGranularity(): + del requested_jobs_dict[job['start_time']] + + #Requested jobs + for start_time in requested_jobs_dict: + requested_nodes_by_start_time[int(start_time)] = \ + requested_jobs_dict[start_time]['hostname'] + #Check if there is any difference between the leases already + #registered in OAR and the requested jobs. + #Difference could be: + #-Lease deleted in the requested jobs + #-Added/removed nodes + #-Newly added lease + + logger.debug("IOTLABSLICES verify_slice_leases \ + requested_nodes_by_start_time %s \ + "% (requested_nodes_by_start_time)) + #Find all deleted leases + start_time_list = \ + list(set(leases_by_start_time.keys()).\ + difference(requested_nodes_by_start_time.keys())) + deleted_leases = [leases_by_start_time[start_time]['lease_id'] \ + for start_time in start_time_list] + + + #Find added or removed nodes in exisiting leases + for start_time in requested_nodes_by_start_time: + logger.debug("IOTLABSLICES verify_slice_leases start_time %s \ + "%( start_time)) + if start_time in current_nodes_reserved_by_start_time: + + if requested_nodes_by_start_time[start_time] == \ + current_nodes_reserved_by_start_time[start_time]: + continue + + else: + update_node_set = \ + set(requested_nodes_by_start_time[start_time]) + added_nodes = \ + update_node_set.difference(\ + current_nodes_reserved_by_start_time[start_time]) + shared_nodes = \ + update_node_set.intersection(\ + current_nodes_reserved_by_start_time[start_time]) + old_nodes_set = \ + set(\ + current_nodes_reserved_by_start_time[start_time]) + removed_nodes = \ + old_nodes_set.difference(\ + requested_nodes_by_start_time[start_time]) + logger.debug("IOTLABSLICES verify_slice_leases \ + shared_nodes %s added_nodes %s removed_nodes %s"\ + %(shared_nodes, added_nodes,removed_nodes )) + #If the lease is modified, delete it before + #creating it again. + #Add the deleted lease job id in the list + #WARNING :rescheduling does not work if there is already + # 2 running/scheduled jobs because deleting a job + #takes time SA 18/10/2012 + if added_nodes or removed_nodes: + deleted_leases.append(\ + leases_by_start_time[start_time]['lease_id']) + #Reschedule the job + if added_nodes or shared_nodes: + reschedule_jobs_dict[str(start_time)] = \ + requested_jobs_dict[str(start_time)] + + else: + #New lease + + job = requested_jobs_dict[str(start_time)] + logger.debug("IOTLABSLICES \ + NEWLEASE slice %s job %s" + % (sfa_slice, job)) + self.driver.iotlab_api.AddLeases( + job['hostname'], + sfa_slice, int(job['start_time']), + int(job['duration'])) + + #Deleted leases are the ones with lease id not declared in the Rspec + if deleted_leases: + self.driver.iotlab_api.DeleteLeases(deleted_leases, + sfa_slice['user']['uid']) + logger.debug("IOTLABSLICES \ + verify_slice_leases slice %s deleted_leases %s" + % (sfa_slice, deleted_leases)) + + if reschedule_jobs_dict: + for start_time in reschedule_jobs_dict: + job = reschedule_jobs_dict[start_time] + self.driver.iotlab_api.AddLeases( + job['hostname'], + sfa_slice, int(job['start_time']), + int(job['duration'])) + return leases + + def verify_slice_nodes(self, sfa_slice, requested_slivers, peer): + """Check for wanted and unwanted nodes in the slice. + + Removes nodes and associated leases that the user does not want anymore + by deleteing the associated job in OAR (DeleteSliceFromNodes). + Returns the nodes' hostnames that are going to be in the slice. + + :param sfa_slice: slice record. Must contain node_ids and list_node_ids. + + :param requested_slivers: list of requested nodes' hostnames. + :param peer: unused so far. + + :type sfa_slice: dict + :type requested_slivers: list + :type peer: string + + :returns: list requested nodes hostnames + :rtype: list + + .. warning:: UNUSED SQA 24/07/13 + .. seealso:: DeleteSliceFromNodes + .. todo:: check what to do with the peer? Can not remove peer nodes from + slice here. Anyway, in this case, the peer should have gotten the + remove request too. + + """ + current_slivers = [] + deleted_nodes = [] + + if 'node_ids' in sfa_slice: + nodes = self.driver.iotlab_api.GetNodes( + sfa_slice['list_node_ids'], + ['hostname']) + current_slivers = [node['hostname'] for node in nodes] + + # remove nodes not in rspec + deleted_nodes = list(set(current_slivers). + difference(requested_slivers)) + + logger.debug("IOTLABSLICES \tverify_slice_nodes slice %s\ + \r\n \r\n deleted_nodes %s" + % (sfa_slice, deleted_nodes)) + + if deleted_nodes: + #Delete the entire experience + self.driver.iotlab_api.DeleteSliceFromNodes(sfa_slice) + return nodes + + def verify_slice(self, slice_hrn, slice_record, sfa_peer): + """Ensures slice record exists. + + The slice record must exist either in Iotlab or in the other + federated testbed (sfa_peer). If the slice does not belong to Iotlab, + check if the user already exists in LDAP. In this case, adds the slice + to the sfa DB and associates its LDAP user. + + :param slice_hrn: slice's name + :param slice_record: sfa record of the slice + :param sfa_peer: name of the peer authority if any.(not Iotlab). + + :type slice_hrn: string + :type slice_record: dictionary + :type sfa_peer: string + + .. seealso:: AddSlice + + + """ + + slicename = slice_hrn + # check if slice belongs to Iotlab + slices_list = self.driver.iotlab_api.GetSlices( + slice_filter=slicename, slice_filter_type='slice_hrn') + + sfa_slice = None + + if slices_list: + for sl in slices_list: + + logger.debug("SLABSLICE \t verify_slice slicename %s \ + slices_list %s sl %s \r slice_record %s" + % (slicename, slices_list, sl, slice_record)) + sfa_slice = sl + sfa_slice.update(slice_record) + + else: + #Search for user in ldap based on email SA 14/11/12 + ldap_user = self.driver.iotlab_api.ldap.LdapFindUser(\ + slice_record['user']) + logger.debug(" IOTLABSLICES \tverify_slice Oups \ + slice_record %s sfa_peer %s ldap_user %s" + % (slice_record, sfa_peer, ldap_user)) + #User already registered in ldap, meaning user should be in SFA db + #and hrn = sfa_auth+ uid + sfa_slice = {'hrn': slicename, + 'node_list': [], + 'authority': slice_record['authority'], + 'gid': slice_record['gid'], + 'slice_id': slice_record['record_id'], + 'reg-researchers': slice_record['reg-researchers'], + 'peer_authority': str(sfa_peer) + } + + if ldap_user: + hrn = self.driver.iotlab_api.root_auth + '.' + ldap_user['uid'] + user = self.driver.get_user_record(hrn) + + logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s" + % (hrn, user)) + + # add the external slice to the local SFA iotlab DB + if sfa_slice: + self.driver.iotlab_api.AddSlice(sfa_slice, user) + + logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK") + return sfa_slice + + + def verify_persons(self, slice_hrn, slice_record, users, options={}): + """Ensures the users in users list exist and are enabled in LDAP. Adds + person if needed. + + Checking that a user exist is based on the user's email. If the user is + still not found in the LDAP, it means that the user comes from another + federated testbed. In this case an account has to be created in LDAP + so as to enable the user to use the testbed, since we trust the testbed + he comes from. This is done by calling AddPerson. + + :param slice_hrn: slice name + :param slice_record: record of the slice_hrn + :param users: users is a record list. Records can either be + local records or users records from known and trusted federated + sites.If the user is from another site that iotlab doesn't trust yet, + then Resolve will raise an error before getting to create_sliver. + + :type slice_hrn: string + :type slice_record: string + :type users: list + + .. seealso:: AddPerson + .. note:: Removed unused peer and sfa_peer parameters. SA 18/07/13. + + + """ + #TODO SA 21/08/12 verify_persons Needs review + + logger.debug("IOTLABSLICES \tverify_persons \tslice_hrn %s \ + \t slice_record %s\r\n users %s \t " + % (slice_hrn, slice_record, users)) + users_by_id = {} + + users_by_email = {} + #users_dict : dict whose keys can either be the user's hrn or its id. + #Values contains only id and hrn + users_dict = {} + + #First create dicts by hrn and id for each user in the user record list: + for info in users: + if 'slice_record' in info: + slice_rec = info['slice_record'] + if 'user' in slice_rec : + user = slice_rec['user'] + + if 'email' in user: + users_by_email[user['email']] = user + users_dict[user['email']] = user + + logger.debug("SLABSLICE.PY \t verify_person \ + users_dict %s \r\n user_by_email %s \r\n \ + \tusers_by_id %s " + % (users_dict, users_by_email, users_by_id)) + + existing_user_ids = [] + existing_user_emails = [] + existing_users = [] + # Check if user is in Iotlab LDAP using its hrn. + # Assuming Iotlab is centralised : one LDAP for all sites, + # user's record_id unknown from LDAP + # LDAP does not provide users id, therefore we rely on email to find the + # user in LDAP + + if users_by_email: + #Construct the list of filters (list of dicts) for GetPersons + filter_user = [users_by_email[email] for email in users_by_email] + #Check user i in LDAP with GetPersons + #Needed because what if the user has been deleted in LDAP but + #is still in SFA? + existing_users = self.driver.iotlab_api.GetPersons(filter_user) + logger.debug(" \r\n SLABSLICE.PY \tverify_person filter_user \ + %s existing_users %s " + % (filter_user, existing_users)) + #User is in iotlab LDAP + if existing_users: + for user in existing_users: + users_dict[user['email']].update(user) + existing_user_emails.append( + users_dict[user['email']]['email']) + + + # User from another known trusted federated site. Check + # if a iotlab account matching the email has already been created. + else: + req = 'mail=' + if isinstance(users, list): + req += users[0]['email'] + else: + req += users['email'] + ldap_reslt = self.driver.iotlab_api.ldap.LdapSearch(req) + + if ldap_reslt: + logger.debug(" SLABSLICE.PY \tverify_person users \ + USER already in Iotlab \t ldap_reslt %s \ + " % (ldap_reslt)) + existing_users.append(ldap_reslt[1]) + + else: + #User not existing in LDAP + logger.debug("SLABSLICE.PY \tverify_person users \ + not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \ + ldap_reslt %s " % (users, ldap_reslt)) + + requested_user_emails = users_by_email.keys() + requested_user_hrns = \ + [users_by_email[user]['hrn'] for user in users_by_email] + logger.debug("SLABSLICE.PY \tverify_person \ + users_by_email %s " % (users_by_email)) + + #Check that the user of the slice in the slice record + #matches one of the existing users + try: + if slice_record['PI'][0] in requested_user_hrns: + logger.debug(" SLABSLICE \tverify_person ['PI']\ + slice_record %s" % (slice_record)) + + except KeyError: + pass + + # users to be added, removed or updated + #One user in one iotlab slice : there should be no need + #to remove/ add any user from/to a slice. + #However a user from SFA which is not registered in Iotlab yet + #should be added to the LDAP. + added_user_emails = set(requested_user_emails).\ + difference(set(existing_user_emails)) + + + #self.verify_keys(existing_slice_users, updated_users_list, \ + #peer, append) + + added_persons = [] + # add new users + #requested_user_email is in existing_user_emails + if len(added_user_emails) == 0: + slice_record['login'] = users_dict[requested_user_emails[0]]['uid'] + logger.debug(" SLABSLICE \tverify_person QUICK DIRTY %s" + % (slice_record)) + + for added_user_email in added_user_emails: + added_user = users_dict[added_user_email] + logger.debug(" IOTLABSLICES \r\n \r\n \t verify_person \ + added_user %s" % (added_user)) + person = {} + person['peer_person_id'] = None + k_list = ['first_name', 'last_name', 'person_id'] + for k in k_list: + if k in added_user: + person[k] = added_user[k] + + person['pkey'] = added_user['keys'][0] + person['mail'] = added_user['email'] + person['email'] = added_user['email'] + person['key_ids'] = added_user.get('key_ids', []) + + ret = self.driver.iotlab_api.AddPerson(person) + if 'uid' in ret: + # meaning bool is True and the AddPerson was successful + person['uid'] = ret['uid'] + slice_record['login'] = person['uid'] + else: + # error message in ret + logger.debug(" IOTLABSLICES ret message %s" %(ret)) + + logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person\ + person %s" % (person)) + #Update slice_Record with the id now known to LDAP + + + added_persons.append(person) + return added_persons + + + def verify_keys(self, persons, users, peer, options={}): + """ + .. warning:: unused + """ + # existing keys + key_ids = [] + for person in persons: + key_ids.extend(person['key_ids']) + keylist = self.driver.iotlab_api.GetKeys(key_ids, ['key_id', 'key']) + + keydict = {} + for key in keylist: + keydict[key['key']] = key['key_id'] + existing_keys = keydict.keys() + + persondict = {} + for person in persons: + persondict[person['email']] = person + + # add new keys + requested_keys = [] + updated_persons = [] + users_by_key_string = {} + for user in users: + user_keys = user.get('keys', []) + updated_persons.append(user) + for key_string in user_keys: + users_by_key_string[key_string] = user + requested_keys.append(key_string) + if key_string not in existing_keys: + key = {'key': key_string, 'key_type': 'ssh'} + #try: + ##if peer: + #person = persondict[user['email']] + #self.driver.iotlab_api.UnBindObjectFromPeer( + # 'person',person['person_id'], + # peer['shortname']) + ret = self.driver.iotlab_api.AddPersonKey( + user['email'], key) + #if peer: + #key_index = user_keys.index(key['key']) + #remote_key_id = user['key_ids'][key_index] + #self.driver.iotlab_api.BindObjectToPeer('key', \ + #key['key_id'], peer['shortname'], \ + #remote_key_id) + + #finally: + #if peer: + #self.driver.iotlab_api.BindObjectToPeer('person', \ + #person['person_id'], peer['shortname'], \ + #user['person_id']) + + # remove old keys (only if we are not appending) + append = options.get('append', True) + if append is False: + removed_keys = set(existing_keys).difference(requested_keys) + for key in removed_keys: + #if peer: + #self.driver.iotlab_api.UnBindObjectFromPeer('key', \ + #key, peer['shortname']) + + user = users_by_key_string[key] + self.driver.iotlab_api.DeleteKey(user, key) + + return diff --git a/sfa/managers/aggregate_manager.py b/sfa/managers/aggregate_manager.py index 0ba512e8..792f8243 100644 --- a/sfa/managers/aggregate_manager.py +++ b/sfa/managers/aggregate_manager.py @@ -47,7 +47,7 @@ class AggregateManager: return version_string def GetVersion(self, api, options): - xrn=Xrn(api.hrn, type='authority') + xrn=Xrn(api.hrn, type='authority+am') version = version_core() cred_types = [{'geni_type': 'geni_sfa', 'geni_version': str(i)} for i in range(4)[-2:]] geni_api_versions = ApiVersions().get_versions() diff --git a/sfa/managers/slice_manager.py b/sfa/managers/slice_manager.py index 67b906e7..746bac6b 100644 --- a/sfa/managers/slice_manager.py +++ b/sfa/managers/slice_manager.py @@ -48,7 +48,7 @@ class SliceManager: ad_rspec_versions.append(rspec_version.to_dict()) if rspec_version.content_type in ['*', 'request']: request_rspec_versions.append(rspec_version.to_dict()) - xrn=Xrn(api.hrn, 'authority+sa') + xrn=Xrn(api.hrn, 'authority+sm') version_more = { 'interface':'slicemgr', 'sfa': 2, diff --git a/sfa/managers/v2_to_v3_adapter.py b/sfa/managers/v2_to_v3_adapter.py index 2d8d4b7d..15a8cd83 100644 --- a/sfa/managers/v2_to_v3_adapter.py +++ b/sfa/managers/v2_to_v3_adapter.py @@ -88,7 +88,7 @@ class V2ToV3Adapter: (slice_urn, rspec_string, expiration, options) = args slice_hrn, type = urn_to_hrn(slice_urn) creds = [] - users = options.get('geni_users', []) + users = options.get('sfa_users', []) manifest_string = getattr(self.driver, "create_sliver")(slice_urn, slice_hrn, creds, rspec_string, users, options) # slivers allocation diff --git a/sfa/methods/CreateGid.py b/sfa/methods/CreateGid.py index eb001a1c..0a2785f5 100644 --- a/sfa/methods/CreateGid.py +++ b/sfa/methods/CreateGid.py @@ -7,13 +7,13 @@ from sfa.trust.credential import Credential class CreateGid(Method): """ - Create a signed credential for the s object with the registry. In addition to being stored in the + Create a signed credential for the object with the registry. In addition to being stored in the SFA database, the appropriate records will also be created in the PLC databases + @param cred credential string @param xrn urn or hrn of certificate owner @param cert caller's certificate - @param cred credential string @return gid string representation """ @@ -39,9 +39,6 @@ class CreateGid(Method): #log the call origin_hrn = Credential(string=valid_creds[0]).get_gid_caller().get_hrn() - - # log - origin_hrn = Credential(string=valid_creds[0]).get_gid_caller().get_hrn() self.api.logger.info("interface: %s\tcaller-hrn: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, origin_hrn, xrn, self.name)) return self.api.manager.CreateGid(self.api, xrn, cert) diff --git a/sfa/nitos/nitosaggregate.py b/sfa/nitos/nitosaggregate.py index 2e59d2e0..bb7c56d6 100644 --- a/sfa/nitos/nitosaggregate.py +++ b/sfa/nitos/nitosaggregate.py @@ -18,7 +18,7 @@ from sfa.rspecs.elements.granularity import Granularity from sfa.rspecs.elements.channel import Channel from sfa.rspecs.version_manager import VersionManager -from sfa.nitos.nitosxrn import NitosXrn, hostname_to_urn, hrn_to_nitos_slicename, slicename_to_hrn +from sfa.nitos.nitosxrn import NitosXrn, hostname_to_urn, hrn_to_nitos_slicename, slicename_to_hrn, channel_to_urn from sfa.planetlab.vlink import get_tc_rate from sfa.planetlab.topology import Topology @@ -163,6 +163,7 @@ class NitosAggregate: rspec_channel['channel_num'] = channel_number rspec_channel['start_time'] = channel['start_time'] rspec_channel['duration'] = (int(channel['end_time']) - int(channel['start_time'])) / int(grain) + rspec_channel['component_id'] = channel_to_urn(self.driver.hrn, self.driver.testbedInfo['name'], channel_number) # retreive slicename for slc in slices: @@ -238,6 +239,7 @@ class NitosAggregate: rspec_channel['channel_num'] = channel['channel'] rspec_channel['frequency'] = channel['frequency'] rspec_channel['standard'] = channel['modulation'] + rspec_channel['component_id'] = channel_to_urn(self.driver.hrn, self.driver.testbedInfo['name'], channel['channel']) rspec_channels.append(rspec_channel) return rspec_channels @@ -275,8 +277,8 @@ class NitosAggregate: rspec.version.add_channels(channels) if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'resources': - leases, channels = self.get_leases_and_channels(slice, slice_xrn) - rspec.version.add_leases(leases, channels) + leases_channels = self.get_leases_and_channels(slice, slice_xrn) + rspec.version.add_leases(leases_channels) return rspec.toxml() diff --git a/sfa/nitos/nitosdriver.py b/sfa/nitos/nitosdriver.py index aa87bcc7..da35ca35 100644 --- a/sfa/nitos/nitosdriver.py +++ b/sfa/nitos/nitosdriver.py @@ -612,8 +612,17 @@ class NitosDriver (Driver): # add/remove leases (nodes and channels) # a lease in Nitos RSpec case is a reservation of nodes and channels grouped by (slice,timeslot) - rspec_requested_nodes, rspec_requested_channels = rspec.version.get_leases() - + rspec_requested_leases = rspec.version.get_leases() + rspec_requested_nodes = [] + rspec_requested_channels = [] + for lease in rspec_requested_leases: + if lease['type'] == 'node': + lease.pop('type', None) + rspec_requested_nodes.append(lease) + else: + lease.pop('type', None) + rspec_requested_channels.append(lease) + nodes = slices.verify_slice_leases_nodes(slice, rspec_requested_nodes) channels = slices.verify_slice_leases_channels(slice, rspec_requested_channels) diff --git a/sfa/nitos/nitosslices.py b/sfa/nitos/nitosslices.py index 2edd2092..3eac8aa3 100644 --- a/sfa/nitos/nitosslices.py +++ b/sfa/nitos/nitosslices.py @@ -7,7 +7,7 @@ from sfa.util.xrn import Xrn, get_leaf, get_authority, urn_to_hrn from sfa.rspecs.rspec import RSpec -from sfa.nitos.nitosxrn import NitosXrn, hrn_to_nitos_slicename, xrn_to_hostname +from sfa.nitos.nitosxrn import NitosXrn, hrn_to_nitos_slicename, xrn_to_hostname, xrn_to_channel MAXINT = 2L**31-1 @@ -96,7 +96,7 @@ class NitosSlices: slice_name = hrn_to_nitos_slicename(channel['slice_id']) if slice_name != slice['slice_name']: continue - channel_num = channel['channel_num'] + channel_num = xrn_to_channel(channel['component_id']) nitos_channel = self.driver.filter_nitos_results(nitos_channels, {'channel': channel_num})[0] # fill the requested channel with nitos ids requested_channel['slice_id'] = slice['slice_id'] diff --git a/sfa/nitos/nitosxrn.py b/sfa/nitos/nitosxrn.py index 442e48b0..47d8aadf 100644 --- a/sfa/nitos/nitosxrn.py +++ b/sfa/nitos/nitosxrn.py @@ -23,6 +23,12 @@ def hrn_to_nitos_authname (hrn): return NitosXrn(xrn=hrn,type='any').nitos_authname() def xrn_to_hostname(hrn): return Xrn.unescape(NitosXrn(xrn=hrn, type='node').get_leaf()) +def channel_to_hrn (auth, login_base, channel): + return NitosXrn(auth=auth+'.'+login_base, channel=channel).get_hrn() +def channel_to_urn (auth, login_base, channel): + return NitosXrn(auth=auth+'.'+login_base, channel=channel).get_urn() +def xrn_to_channel(hrn): + return Xrn.unescape(NitosXrn(xrn=hrn, type='channel').get_leaf()) class NitosXrn (Xrn): @@ -30,7 +36,7 @@ class NitosXrn (Xrn): def site_hrn (auth, login_base): return '.'.join([auth,login_base]) - def __init__ (self, auth=None, hostname=None, slicename=None, email=None, interface=None, **kwargs): + def __init__ (self, auth=None, hostname=None, slicename=None, email=None, interface=None, channel=None, **kwargs): #def hostname_to_hrn(auth_hrn, login_base, hostname): if hostname is not None: self.type='node' @@ -54,6 +60,10 @@ class NitosXrn (Xrn): self.type = 'interface' self.hrn = auth + '.' + interface self.hrn_to_urn() + elif channel is not None: + self.type='channel' + self.hrn=".".join([auth] + [channel]) + self.hrn_to_urn() else: Xrn.__init__ (self,**kwargs) diff --git a/sfa/planetlab/plaggregate.py b/sfa/planetlab/plaggregate.py index d869cc1a..fdd70e5f 100644 --- a/sfa/planetlab/plaggregate.py +++ b/sfa/planetlab/plaggregate.py @@ -1,6 +1,6 @@ #!/usr/bin/python from collections import defaultdict -from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn +from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn, get_authority, get_leaf from sfa.util.sfatime import utcparse, datetime_to_string from sfa.util.sfalogging import logger from sfa.util.faults import SliverDoesNotExist @@ -18,7 +18,7 @@ from sfa.rspecs.elements.lease import Lease from sfa.rspecs.elements.granularity import Granularity from sfa.rspecs.version_manager import VersionManager -from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn +from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn, xrn_to_ext_slicename, top_auth from sfa.planetlab.vlink import get_tc_rate from sfa.planetlab.topology import Topology from sfa.storage.alchemy import dbsession @@ -129,7 +129,13 @@ class PlAggregate: except ValueError: pass else: - names.add(xrn.pl_slicename()) + slice_hrn = xrn.get_hrn() + top_auth_hrn = top_auth(slice_hrn) + if top_auth_hrn == self.driver.hrn: + slice_name = hrn_to_pl_slicename(slice_hrn) + else: + slice_name = xrn_to_ext_slicename(slice_hrn) + names.add(slice_name) filter = {} if names: @@ -141,7 +147,7 @@ class PlAggregate: if not slices: return [] slice = slices[0] - slice['hrn'] = PlXrn(auth=self.driver.hrn, slicename=slice['name']).hrn + slice['hrn'] = slice_hrn # get sliver users persons = [] @@ -165,10 +171,10 @@ class PlAggregate: # construct user key info users = [] for person in persons: - name = person['email'][0:person['email'].index('@')] + person_urn = hrn_to_urn(self.driver.shell.GetPersonHrn(int(person['person_id'])), 'user') user = { 'login': slice['name'], - 'user_urn': Xrn('%s.%s' % (self.driver.hrn, name), type='user').urn, + 'user_urn': person_urn, 'keys': [keys[k_id]['key'] for k_id in person['key_ids'] if k_id in keys] } users.append(user) @@ -356,8 +362,8 @@ class PlAggregate: site_id=lease['site_id'] site=sites_dict[site_id] - rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], lease['hostname']) - slice_hrn = slicename_to_hrn(self.driver.hrn, lease['name']) + rspec_lease['component_id'] = hrn_to_urn(self.driver.shell.GetNodeHrn(lease['hostname']), 'node') + slice_hrn = self.driver.shell.GetSliceHrn(lease['slice_id']) slice_urn = hrn_to_urn(slice_hrn, 'slice') rspec_lease['slice_id'] = slice_urn rspec_lease['start_time'] = lease['t_from'] diff --git a/sfa/planetlab/pldriver.py b/sfa/planetlab/pldriver.py index 8b02d349..79c61f0a 100644 --- a/sfa/planetlab/pldriver.py +++ b/sfa/planetlab/pldriver.py @@ -25,7 +25,7 @@ from sfa.planetlab.plshell import PlShell import sfa.planetlab.peers as peers from sfa.planetlab.plaggregate import PlAggregate from sfa.planetlab.plslices import PlSlices -from sfa.planetlab.plxrn import PlXrn, slicename_to_hrn, hostname_to_hrn, hrn_to_pl_slicename, xrn_to_hostname +from sfa.planetlab.plxrn import PlXrn, slicename_to_hrn, hostname_to_hrn, hrn_to_pl_slicename, xrn_to_hostname, xrn_to_ext_slicename, top_auth def list_to_dict(recs, key): @@ -646,7 +646,7 @@ class PlDriver (Driver): # ensure slice record exists slice = slices.verify_slice(xrn.hrn, slice_record, peer, sfa_peer, expiration=expiration, options=options) # ensure person records exists - #persons = slices.verify_persons(xrn.hrn, slice, users, peer, sfa_peer, options=options) + persons = slices.verify_persons(xrn.hrn, slice, users, peer, sfa_peer, options=options) # ensure slice attributes exists slices.verify_slice_attributes(slice, requested_attributes, options=options) @@ -676,7 +676,7 @@ class PlDriver (Driver): peer = slices.get_peer(slice['hrn']) sfa_peer = slices.get_sfa_peer(slice['hrn']) users = options.get('geni_users', []) - persons = slices.verify_persons(None, slice, users, peer, sfa_peer, options=options) + persons = slices.verify_persons(slice['hrn'], slice, users, peer, sfa_peer, options=options) slices.handle_peer(None, None, persons, peer) # update sliver allocation states and set them to geni_provisioned sliver_ids = [sliver['sliver_id'] for sliver in slivers] @@ -692,23 +692,31 @@ class PlDriver (Driver): slivers = aggregate.get_slivers(urns) if slivers: slice_id = slivers[0]['slice_id'] + slice_name = slivers[0]['name'] node_ids = [] sliver_ids = [] for sliver in slivers: node_ids.append(sliver['node_id']) sliver_ids.append(sliver['sliver_id']) + # leases + leases = self.shell.GetLeases({'name': slice_name}) + leases_ids = [lease['lease_id'] for lease in leases ] + # determine if this is a peer slice # xxx I wonder if this would not need to use PlSlices.get_peer instead # in which case plc.peers could be deprecated as this here # is the only/last call to this last method in plc.peers - slice_hrn = PlXrn(auth=self.hrn, slicename=slivers[0]['name']).get_hrn() + #slice_hrn = PlXrn(auth=self.hrn, slice_name).get_hrn() + slice_hrn = self.shell.GetSliceHrn(int(slice_id)) peer = peers.get_peer(self, slice_hrn) try: if peer: self.shell.UnBindObjectFromPeer('slice', slice_id, peer) self.shell.DeleteSliceFromNodes(slice_id, node_ids) + if len(leases_ids) > 0: + self.shell.DeleteLeases(leases_ids) # delete sliver allocation states SliverAllocation.delete_allocations(sliver_ids) @@ -759,8 +767,12 @@ class PlDriver (Driver): # set the 'enabled' tag to 0 def shutdown (self, xrn, options={}): - xrn = PlXrn(xrn=xrn, type='slice') - slicename = xrn.pl_slicename() + hrn = urn_to_hrn(xrn) + top_auth_hrn = top_auth(hrn) + if top_auth_hrn == self.hrn: + slicename = hrn_to_pl_slicename(hrn) + else: + slicename = xrn_to_ext_slicename(hrn) slices = self.shell.GetSlices({'name': slicename}, ['slice_id']) if not slices: raise RecordNotFound(slice_hrn) diff --git a/sfa/planetlab/plshell.py b/sfa/planetlab/plshell.py index f42af880..15334db0 100644 --- a/sfa/planetlab/plshell.py +++ b/sfa/planetlab/plshell.py @@ -25,7 +25,10 @@ class PlShell: 'GetSites','GetNodes', # Lease management methods 'GetLeases', 'GetLeaseGranularity', 'DeleteLeases','UpdateLeases', - 'AddLeases' + 'AddLeases', + # HRN management methods + 'SetPersonHrn', 'GetPersonHrn', 'SetSliceHrn', 'GetSliceHrn', + 'SetNodeHrn', 'GetNodeHrn' ] # support for other names - this is experimental alias_calls = { 'get_authorities':'GetSites', diff --git a/sfa/planetlab/plslices.py b/sfa/planetlab/plslices.py index fedb64bc..65e2164a 100644 --- a/sfa/planetlab/plslices.py +++ b/sfa/planetlab/plslices.py @@ -8,7 +8,7 @@ from sfa.util.xrn import Xrn, get_leaf, get_authority, urn_to_hrn from sfa.rspecs.rspec import RSpec from sfa.planetlab.vlink import VLink from sfa.planetlab.topology import Topology -from sfa.planetlab.plxrn import PlXrn, hrn_to_pl_slicename, xrn_to_hostname +from sfa.planetlab.plxrn import PlXrn, hrn_to_pl_slicename, xrn_to_hostname, xrn_to_ext_slicename, hrn_to_ext_loginbase, top_auth from sfa.storage.model import SliverAllocation from sfa.storage.alchemy import dbsession @@ -169,7 +169,12 @@ class PlSlices: requested_leases = [] for lease in rspec_requested_leases: requested_lease = {} - slice_name = hrn_to_pl_slicename(lease['slice_id']) + slice_hrn, _ = urn_to_hrn(lease['slice_id']) + top_auth_hrn = top_auth(slice_hrn) + if top_auth_hrn == self.driver.hrn: + slice_name = hrn_to_pl_slicename(lease['slice_id']) + else: + slice_name = xrn_to_ext_slicename(lease['slice_id']) if slice_name != slice['name']: continue elif Xrn(lease['component_id']).get_authority_urn().split(':')[0] != self.driver.hrn: @@ -374,15 +379,20 @@ class PlSlices: def verify_site(self, slice_xrn, slice_record={}, peer=None, sfa_peer=None, options={}): (slice_hrn, type) = urn_to_hrn(slice_xrn) - site_hrn = get_authority(slice_hrn) - # login base can't be longer than 20 characters - slicename = hrn_to_pl_slicename(slice_hrn) - authority_name = slicename.split('_')[0] - login_base = authority_name[:20] + top_auth_hrn = top_auth(slice_hrn) + if top_auth_hrn == self.driver.hrn: + # login base can't be longer than 20 characters + slicename = hrn_to_pl_slicename(slice_hrn) + authority_name = slicename.split('_')[0] + login_base = authority_name[:20] + else: + login_base = hrn_to_ext_loginbase(slice_hrn) + authority_name = login_base + sites = self.driver.shell.GetSites(login_base) if not sites: # create new site record - site = {'name': 'geni.%s' % authority_name, + site = {'name': 'sfa.%s' % authority_name, 'abbreviated_name': authority_name, 'login_base': login_base, 'max_slices': 100, @@ -394,7 +404,7 @@ class PlSlices: site['site_id'] = self.driver.shell.AddSite(site) # exempt federated sites from monitor policies self.driver.shell.AddSiteTag(site['site_id'], 'exempt_site_until', "20200101") - + # # is this still necessary? # # add record to the local registry # if sfa_peer and slice_record: @@ -405,30 +415,44 @@ class PlSlices: site = sites[0] if peer: # unbind from peer so we can modify if necessary. Will bind back later - self.driver.shell.UnBindObjectFromPeer('site', site['site_id'], peer['shortname']) - - return site + self.driver.shell.UnBindObjectFromPeer('site', site['site_id'], peer['shortname']) + + return site + def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer, expiration, options={}): - slicename = hrn_to_pl_slicename(slice_hrn) - parts = slicename.split("_") - login_base = parts[0] + top_auth_hrn = top_auth(slice_hrn) + if top_auth_hrn == self.driver.hrn: + slicename = hrn_to_pl_slicename(slice_hrn) + parts = slicename.split("_") + login_base = parts[0] + else: + login_base = hrn_to_ext_loginbase(slice_hrn) + slicename = xrn_to_ext_slicename(slice_hrn) + slices = self.driver.shell.GetSlices([slicename]) expires = int(datetime_to_epoch(utcparse(expiration))) if not slices: slice = {'name': slicename, - 'url': 'No Url', - 'description': 'No Description'} + 'url': slice_record.get('url', slice_hrn), + 'description': slice_record.get('description', slice_hrn)} # add the slice slice['slice_id'] = self.driver.shell.AddSlice(slice) slice['node_ids'] = [] slice['person_ids'] = [] + # set the slice HRN + self.driver.shell.SetSliceHrn(int(slice['slice_id']), slice_hrn) + if peer and slice_record: slice['peer_slice_id'] = slice_record.get('slice_id', None) # set the expiration self.driver.shell.UpdateSlice(slice['slice_id'], {'expires': expires}) else: slice = slices[0] + # Check slice HRN + if self.driver.shell.GetSliceHrn(slice['slice_id']) != slice_hrn: + self.driver.shell.SetSliceHrn(slice['slice_id'], slice_hrn) + if peer and slice_record: slice['peer_slice_id'] = slice_record.get('slice_id', None) # unbind from peer so we can modify if necessary. Will bind back later @@ -444,17 +468,23 @@ class PlSlices: def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, options={}): users_by_email = {} users_by_site = defaultdict(list) - users_dict = {} + users_dict = {} for user in users: user['urn'] = user['urn'].lower() hrn, type = urn_to_hrn(user['urn']) username = get_leaf(hrn) - login_base = PlXrn(xrn=user['urn']).pl_login_base() user['username'] = username - user['site'] = login_base + top_auth_hrn = top_auth(hrn) + + if top_auth_hrn == self.driver.hrn: + login_base = PlXrn(xrn=user['urn']).pl_login_base() + else: + login_base = hrn_to_ext_loginbase(hrn) + + user['site'] = login_base if 'email' in user: - user['email'] = user['email'].lower() + user['email'] = user['email'].lower() users_by_email[user['email']] = user users_dict[user['email']] = user else: @@ -468,14 +498,15 @@ class PlSlices: if users_by_site: for login_base in users_by_site: users = users_by_site[login_base] - for user in users: - existing_user_ids_filter.append(user['username']+'@geni.net') - if existing_user_ids_filter: + for user in users: + existing_user_ids_filter.append(user['username']+'@geni.net') + + if existing_user_ids_filter: # get existing users by email - existing_users = self.driver.shell.GetPersons({'email': existing_user_ids_filter}, + existing_users = self.driver.shell.GetPersons({'email': existing_user_ids_filter}, ['person_id', 'key_ids', 'email']) existing_user_ids.extend([user['email'] for user in existing_users]) - + if users_by_site: # get a list of user sites (based on requeste user urns site_list = self.driver.shell.GetSites(users_by_site.keys(), \ @@ -487,7 +518,7 @@ class PlSlices: sites[site['site_id']] = site site_user_ids.extend(site['person_ids']) - existing_site_persons_list = self.driver.shell.GetPersons(site_user_ids, + existing_site_persons_list = self.driver.shell.GetPersons(site_user_ids, ['person_id', 'key_ids', 'email', 'site_ids']) # all requested users are either existing users or new (added) users @@ -508,12 +539,12 @@ class PlSlices: break if user_found: break - + if user_found == False: fake_email = requested_user['username'] + '@geni.net' requested_user['email'] = fake_email users_dict[fake_email] = requested_user - + # requested slice users requested_user_ids = users_dict.keys() # existing slice users @@ -521,7 +552,7 @@ class PlSlices: existing_slice_users = self.driver.shell.GetPersons(existing_slice_users_filter, ['person_id', 'key_ids', 'email']) existing_slice_user_ids = [user['email'] for user in existing_slice_users] - + # users to be added, removed or updated added_user_ids = set(requested_user_ids).difference(existing_user_ids) added_slice_user_ids = set(requested_user_ids).difference(existing_slice_user_ids) @@ -543,29 +574,36 @@ class PlSlices: # add new users for added_user_id in added_user_ids: added_user = users_dict[added_user_id] - hrn, type = urn_to_hrn(added_user['urn']) + hrn, type = urn_to_hrn(added_user['urn']) person = { 'first_name': added_user.get('first_name', hrn), 'last_name': added_user.get('last_name', hrn), 'email': added_user_id, - 'peer_person_id': None, - 'keys': [], + #'peer_person_id': None, + #'keys': [], #'key_ids': added_user.get('key_ids', []), } person['person_id'] = self.driver.shell.AddPerson(person) + self.driver.shell.AddRoleToPerson('user', int(person['person_id'])) + # check user HRN + if self.driver.shell.GetPersonHrn(int(person['person_id'])) != hrn: + self.driver.shell.SetPersonHrn(int(person['person_id']), hrn) + if peer: person['peer_person_id'] = added_user['person_id'] added_persons.append(person) - + # enable the account self.driver.shell.UpdatePerson(person['person_id'], {'enabled': True}) - + # add person to site self.driver.shell.AddPersonToSite(added_user_id, added_user['site']) for key_string in added_user.get('keys', []): key = {'key':key_string, 'key_type':'ssh'} key['key_id'] = self.driver.shell.AddPersonKey(person['person_id'], key) + if 'keys' not in person: + person['keys'] = [] person['keys'].append(key) # add the registry record @@ -573,7 +611,7 @@ class PlSlices: # peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': sfa_peer, \ # 'pointer': person['person_id']} # self.registry.register_peer_object(self.credential, peer_dict) - + for added_slice_user_id in added_slice_user_ids.union(added_user_ids): # add person to the slice self.driver.shell.AddPersonToSlice(added_slice_user_id, slice_record['name']) @@ -581,7 +619,7 @@ class PlSlices: # no need to return worry about it getting bound later return added_persons - + def verify_keys(self, persons, users, peer, options={}): # existing keys @@ -591,12 +629,12 @@ class PlSlices: keylist = self.driver.shell.GetKeys(key_ids, ['key_id', 'key']) keydict = {} for key in keylist: - keydict[key['key']] = key['key_id'] + keydict[key['key']] = key['key_id'] existing_keys = keydict.keys() persondict = {} for person in persons: - persondict[person['email']] = person - + persondict[person['email']] = person + # add new keys requested_keys = [] updated_persons = [] @@ -616,14 +654,14 @@ class PlSlices: key_index = user_keys.index(key['key']) remote_key_id = user['key_ids'][key_index] self.driver.shell.BindObjectToPeer('key', key['key_id'], peer['shortname'], remote_key_id) - + finally: if peer: self.driver.shell.BindObjectToPeer('person', person['person_id'], peer['shortname'], user['person_id']) - + # remove old keys (only if we are not appending) append = options.get('append', True) - if append == False: + if append == False: removed_keys = set(existing_keys).difference(requested_keys) for existing_key_id in keydict: if keydict[existing_key_id] in removed_keys: @@ -632,7 +670,7 @@ class PlSlices: self.driver.shell.UnBindObjectFromPeer('key', existing_key_id, peer['shortname']) self.driver.shell.DeleteKey(existing_key_id) except: - pass + pass def verify_slice_attributes(self, slice, requested_slice_attributes, options={}, admin=False): append = options.get('append', True) @@ -648,7 +686,7 @@ class PlSlices: removed_slice_attributes = [] ignored_slice_attribute_names = [] existing_slice_attributes = self.driver.shell.GetSliceTags({'slice_id': slice['slice_id']}) - + # get attributes that should be removed for slice_tag in existing_slice_attributes: if slice_tag['tagname'] in ignored_slice_attribute_names: @@ -667,7 +705,7 @@ class PlSlices: if not attribute_found and not append: removed_slice_attributes.append(slice_tag) - + # get attributes that should be added: for requested_attribute in requested_slice_attributes: # if the requested attribute wasn't found we should add it diff --git a/sfa/planetlab/plxrn.py b/sfa/planetlab/plxrn.py index 22e94181..a8461b53 100644 --- a/sfa/planetlab/plxrn.py +++ b/sfa/planetlab/plxrn.py @@ -1,6 +1,6 @@ # specialized Xrn class for PlanetLab import re -from sfa.util.xrn import Xrn +from sfa.util.xrn import Xrn, get_authority # temporary helper functions to use this module instead of namespace def hostname_to_hrn (auth, login_base, hostname): @@ -21,6 +21,22 @@ def hrn_to_pl_authname (hrn): def xrn_to_hostname(hrn): return Xrn.unescape(PlXrn(xrn=hrn, type='node').get_leaf()) +# helpers to handle external objects created via fedaration +def xrn_to_ext_slicename (xrn): + slice_hrn=PlXrn(xrn=xrn,type='slice').get_hrn() + site_hrn = get_authority(slice_hrn) + login_base = '8'.join(site_hrn.split('.')) + slice_name = '_'.join([login_base, slice_hrn.split('.')[-1]]) + return slice_name + +def hrn_to_ext_loginbase (hrn): + site_hrn = get_authority(hrn) + login_base = '8'.join(site_hrn.split('.'))[:20] + return login_base + +def top_auth (hrn): + return hrn.split('.')[0] + class PlXrn (Xrn): @staticmethod diff --git a/sfa/rspecs/elements/attribute.py b/sfa/rspecs/elements/attribute.py new file mode 100644 index 00000000..740083b7 --- /dev/null +++ b/sfa/rspecs/elements/attribute.py @@ -0,0 +1,9 @@ +from sfa.rspecs.elements.element import Element + +class Attribute(Element): + + fields = [ + 'name', + 'value', + ] + diff --git a/sfa/rspecs/elements/channel.py b/sfa/rspecs/elements/channel.py index 6ed328ed..96680544 100644 --- a/sfa/rspecs/elements/channel.py +++ b/sfa/rspecs/elements/channel.py @@ -10,4 +10,5 @@ class Channel(Element): 'slice_id', 'start_time', 'duration', + 'component_id', ] diff --git a/sfa/rspecs/elements/v3/node.py b/sfa/rspecs/elements/v3/node.py index 1ddaa737..1f20c680 100644 --- a/sfa/rspecs/elements/v3/node.py +++ b/sfa/rspecs/elements/v3/node.py @@ -1,4 +1,4 @@ -from sfa.util.xrn import Xrn +from sfa.util.xrn import Xrn, get_leaf from sfa.util.xml import XpathFilter from sfa.rspecs.elements.node import NodeElement @@ -13,7 +13,6 @@ from sfa.rspecs.elements.v3.services import Services from sfa.rspecs.elements.versions.pgv2SliverType import PGv2SliverType from sfa.rspecs.elements.versions.pgv2Interface import PGv2Interface -from sfa.planetlab.plxrn import xrn_to_hostname class Node: @staticmethod @@ -25,7 +24,7 @@ class Node: node_elems.append(node_elem) # set component name if node.get('component_id'): - component_name = xrn_to_hostname(node['component_id']) + component_name = Xrn.unescape(get_leaf(Xrn(node['component_id']).get_hrn())) node_elem.set('component_name', component_name) # set hardware types if node.get('hardware_types'): diff --git a/sfa/rspecs/elements/versions/slabv1Lease.py b/sfa/rspecs/elements/versions/iotlabv1Lease.py similarity index 75% rename from sfa/rspecs/elements/versions/slabv1Lease.py rename to sfa/rspecs/elements/versions/iotlabv1Lease.py index 2991a490..eebb1ae2 100644 --- a/sfa/rspecs/elements/versions/slabv1Lease.py +++ b/sfa/rspecs/elements/versions/iotlabv1Lease.py @@ -1,20 +1,16 @@ from sfa.util.sfalogging import logger from sfa.util.xml import XpathFilter from sfa.util.xrn import Xrn -from sfa.util.sfatime import utcparse, datetime_to_string, datetime_to_epoch - -#from sfa.rspecs.elements.versions.sfav1PLTag import SFAv1PLTag -#from sfa.rspecs.elements.versions.pgv2Services import PGv2Services from sfa.rspecs.elements.lease import Lease -class Slabv1Lease: +class Iotlabv1Lease: @staticmethod def add_leases(xml, leases): - + network_elems = xml.xpath('//network') if len(network_elems) > 0: network_elem = network_elems[0] @@ -23,10 +19,9 @@ class Slabv1Lease: network_elem = xml.add_element('network', name = network_urn) else: network_elem = xml - - lease_elems = [] + + lease_elems = [] for lease in leases: - lease['start_time'] = datetime_to_string(utcparse(lease['start_time'])) lease_fields = ['lease_id', 'component_id', 'slice_id', 'start_time', 'duration'] lease_elem = network_elem.add_instance('lease', lease, lease_fields) @@ -37,7 +32,7 @@ class Slabv1Lease: def get_leases(xml, filter={}): xpath = '//lease%s | //default:lease%s' % (XpathFilter.xpath(filter), XpathFilter.xpath(filter)) lease_elems = xml.xpath(xpath) - return Slabv1Lease.get_lease_objs(lease_elems) + return Iotlabv1Lease.get_lease_objs(lease_elems) @staticmethod def get_lease_objs(lease_elems): @@ -48,7 +43,7 @@ class Slabv1Lease: for node_elem in node_elems: lease = Lease(lease_elem.attrib, lease_elem) lease['slice_id'] = lease_elem.attrib['slice_id'] - lease['start_time'] = datetime_to_epoch(utcparse(lease_elem.attrib['start_time'])) + lease['start_time'] = lease_elem.attrib['start_time'] lease['duration'] = lease_elem.attrib['duration'] lease['component_id'] = node_elem.attrib['component_id'] leases.append(lease) diff --git a/sfa/rspecs/elements/versions/slabv1Node.py b/sfa/rspecs/elements/versions/iotlabv1Node.py similarity index 64% rename from sfa/rspecs/elements/versions/slabv1Node.py rename to sfa/rspecs/elements/versions/iotlabv1Node.py index 0804c9b8..8d3722df 100644 --- a/sfa/rspecs/elements/versions/slabv1Node.py +++ b/sfa/rspecs/elements/versions/iotlabv1Node.py @@ -7,196 +7,220 @@ from sfa.rspecs.elements.location import Location from sfa.rspecs.elements.hardware_type import HardwareType from sfa.rspecs.elements.element import Element from sfa.rspecs.elements.interface import Interface -from sfa.rspecs.elements.versions.slabv1Sliver import Slabv1Sliver +from sfa.rspecs.elements.versions.iotlabv1Sliver import Iotlabv1Sliver from sfa.util.sfalogging import logger -class SlabNode(NodeElement): +class IotlabNode(NodeElement): #First get the fields already defined in the class Node fields = list(NodeElement.fields) - #Extend it with senslab's specific fields + #Extend it with iotlab's specific fields fields.extend (['archi', 'radio', 'mobile','position']) - -class SlabPosition(Element): + +class IotlabPosition(Element): fields = ['posx', 'posy','posz'] - -class SlabLocation(Location): + +class IotlabLocation(Location): fields = list(Location.fields) fields.extend (['site']) - + + +class IotlabMobility(Element): + """ Class to give information of a node's mobility, and what kind of + mobility it is (train, roomba robot ...) """ + fields = ['mobile', 'mobility_type'] -class Slabv1Node: - +class Iotlabv1Node: + @staticmethod def add_connection_information(xml, ldap_username, sites_set): - """ Adds login and ssh connection info in the network item in - the xml. Does not create the network element, therefore + """ Adds login and ssh connection info in the network item in + the xml. Does not create the network element, therefore should be used after add_nodes, which creates the network item. - + """ - logger.debug(" add_connection_information " ) + logger.debug(" add_connection_information ") #Get network item in the xml - network_elems = xml.xpath('//network') + network_elems = xml.xpath('//network') if len(network_elems) > 0: network_elem = network_elems[0] - slab_network_dict = {} - slab_network_dict['login'] = ldap_username - - slab_network_dict['ssh'] = \ - ['ssh ' + ldap_username + '@'+site+'.senslab.info' \ - for site in sites_set] - network_elem.set('ssh', \ - unicode(slab_network_dict['ssh'])) - network_elem.set('login', unicode( slab_network_dict['login'])) - - + iotlab_network_dict = {} + iotlab_network_dict['login'] = ldap_username + + iotlab_network_dict['ssh'] = \ + ['ssh ' + ldap_username + '@'+site+'.iotlab.info' + for site in sites_set] + network_elem.set('ssh', + unicode(iotlab_network_dict['ssh'])) + network_elem.set('login', unicode(iotlab_network_dict['login'])) + @staticmethod def add_nodes(xml, nodes): + """Adds the nodes to the xml. + + Adds the nodes as well as dedicated iotlab fields to the node xml + element. + + :param xml: the xml being constructed. + :type xml: xml + :param nodes: list of node dict + :type nodes: list + :returns: a list of node elements. + :rtype: list + + """ #Add network item in the xml network_elems = xml.xpath('//network') if len(network_elems) > 0: network_elem = network_elems[0] elif len(nodes) > 0 and nodes[0].get('component_manager_id'): network_urn = nodes[0]['component_manager_id'] - network_elem = xml.add_element('network', \ - name = Xrn(network_urn).get_hrn()) + network_elem = xml.add_element('network', + name=Xrn(network_urn).get_hrn()) else: network_elem = xml - - logger.debug("slabv1Node \t add_nodes nodes %s \r\n "%(nodes[0])) + node_elems = [] #Then add nodes items to the network item in the xml for node in nodes: #Attach this node to the network element - node_fields = ['component_manager_id', 'component_id', 'exclusive',\ - 'boot_state', 'mobile'] + node_fields = ['component_manager_id', 'component_id', 'exclusive', + 'boot_state', 'mobile'] node_elem = network_elem.add_instance('node', node, node_fields) node_elems.append(node_elem) - + #Set the attibutes of this node element - for attribute in node: + for attribute in node: # set component name - if attribute is 'component_id': + if attribute is 'component_name': component_name = node['component_name'] node_elem.set('component_name', component_name) - - # set hardware types, extend fields to add Senslab's architecture + + # set hardware types, extend fields to add Iotlab's architecture #and radio type - + if attribute is 'hardware_types': - for hardware_type in node.get('hardware_types', []): + for hardware_type in node.get('hardware_types', []): fields = HardwareType.fields - fields.extend(['archi','radio']) + fields.extend(['archi', 'radio']) node_elem.add_instance('hardware_types', node, fields) + # set mobility + if attribute is 'mobility': + node_elem.add_instance('mobility', node['mobility'], + IotlabMobility.fields) # set location if attribute is 'location': - node_elem.add_instance('location', node['location'], \ - SlabLocation.fields) + node_elem.add_instance('location', node['location'], + IotlabLocation.fields) + # add granularity of the reservation system #TODO put the granularity in network instead SA 18/07/12 - if attribute is 'granularity' : + if attribute is 'granularity': granularity = node['granularity'] if granularity: - node_elem.add_instance('granularity', \ - granularity, granularity.fields) - - + node_elem.add_instance('granularity', + granularity, granularity.fields) + # set available element if attribute is 'boot_state': if node.get('boot_state').lower() == 'alive': - available_elem = node_elem.add_element('available', \ - now='true') + available_elem = node_elem.add_element('available', + now='true') else: - available_elem = node_elem.add_element('available', \ - now='false') + available_elem = node_elem.add_element('available', + now='false') - #set position + #set position if attribute is 'position': - node_elem.add_instance('position', node['position'], \ - SlabPosition.fields) + node_elem.add_instance('position', node['position'], + IotlabPosition.fields) ## add services - #PGv2Services.add_services(node_elem, node.get('services', [])) + #PGv2Services.add_services(node_elem, node.get('services', [])) # add slivers if attribute is 'slivers': slivers = node.get('slivers', []) if not slivers: # we must still advertise the available sliver types - slivers = Sliver({'type': 'slab-node'}) + slivers = Sliver({'type': 'iotlab-node'}) # we must also advertise the available initscripts #slivers['tags'] = [] - #if node.get('pl_initscripts'): + #if node.get('pl_initscripts'): #for initscript in node.get('pl_initscripts', []): #slivers['tags'].append({'name': 'initscript', \ #'value': initscript['name']}) - - Slabv1Sliver.add_slivers(node_elem, slivers) + + Iotlabv1Sliver.add_slivers(node_elem, slivers) return node_elems - - @staticmethod def get_nodes(xml, filter={}): xpath = '//node%s | //default:node%s' % (XpathFilter.xpath(filter), \ XpathFilter.xpath(filter)) - node_elems = xml.xpath(xpath) - return Slabv1Node.get_node_objs(node_elems) + node_elems = xml.xpath(xpath) + return Iotlabv1Node.get_node_objs(node_elems) - @staticmethod + @staticmethod def get_nodes_with_slivers(xml, sliver_filter={}): xpath = '//node[count(sliver)>0] | \ - //default:node[count(default:sliver) > 0]' - node_elems = xml.xpath(xpath) + //default:node[count(default:sliver) > 0]' + node_elems = xml.xpath(xpath) logger.debug("SLABV1NODE \tget_nodes_with_slivers \ node_elems %s"%(node_elems)) - return Slabv1Node.get_node_objs(node_elems) + return Iotlabv1Node.get_node_objs(node_elems) @staticmethod def get_node_objs(node_elems): nodes = [] for node_elem in node_elems: node = NodeElement(node_elem.attrib, node_elem) - nodes.append(node) + nodes.append(node) if 'component_id' in node_elem.attrib: node['authority_id'] = \ Xrn(node_elem.attrib['component_id']).get_authority_urn() - + # get hardware types hardware_type_elems = node_elem.xpath('./default:hardware_type | \ ./hardware_type') node['hardware_types'] = [hw_type.get_instance(HardwareType) \ for hw_type in hardware_type_elems] - + # get location location_elems = node_elem.xpath('./default:location | ./location') locations = [location_elem.get_instance(Location) \ for location_elem in location_elems] if len(locations) > 0: node['location'] = locations[0] - - + + # get interfaces iface_elems = node_elem.xpath('./default:interface | ./interface') node['interfaces'] = [iface_elem.get_instance(Interface) \ for iface_elem in iface_elems] + # get position + position_elems = node_elem.xpath('./default:position | ./position') + if position_elems: + position_elem = position_elems[0] + node['position'] = position_elem.get_instance(IotlabPosition) + # get services #node['services'] = PGv2Services.get_services(node_elem) # get slivers - node['slivers'] = Slabv1Sliver.get_slivers(node_elem) + node['slivers'] = Iotlabv1Sliver.get_slivers(node_elem) available_elems = node_elem.xpath('./default:available | \ ./available') if len(available_elems) > 0 and 'name' in available_elems[0].attrib: - if available_elems[0].attrib.get('now', '').lower() == 'true': + if available_elems[0].attrib.get('now', '').lower() == 'true': node['boot_state'] = 'boot' - else: - node['boot_state'] = 'disabled' - + else: + node['boot_state'] = 'disabled' + logger.debug("SLABV1NODE \tget_nodes_objs \ #nodes %s"%(nodes)) return nodes @@ -204,7 +228,7 @@ class Slabv1Node: @staticmethod def add_slivers(xml, slivers): - logger.debug("SLABv1NODE \tadd_slivers ") + logger.debug("Iotlabv1NODE \tadd_slivers ") component_ids = [] for sliver in slivers: filter_sliver = {} @@ -213,23 +237,23 @@ class Slabv1Node: sliver = {} elif 'component_id' in sliver and sliver['component_id']: filter_sliver['component_id'] = '*%s*' % sliver['component_id'] - if not filter_sliver: + if not filter_sliver: continue - nodes = Slabv1Node.get_nodes(xml, filter_sliver) + nodes = Iotlabv1Node.get_nodes(xml, filter_sliver) if not nodes: continue node = nodes[0] - Slabv1Sliver.add_slivers(node, sliver) + Iotlabv1Sliver.add_slivers(node, sliver) @staticmethod def remove_slivers(xml, hostnames): for hostname in hostnames: - nodes = Slabv1Node.get_nodes(xml, \ + nodes = Iotlabv1Node.get_nodes(xml, \ {'component_id': '*%s*' % hostname}) for node in nodes: - slivers = Slabv1Sliver.get_slivers(node.element) + slivers = Iotlabv1Sliver.get_slivers(node.element) for sliver in slivers: - node.element.remove(sliver.element) + node.element.remove(sliver.element) + + - - diff --git a/sfa/rspecs/elements/versions/slabv1Sliver.py b/sfa/rspecs/elements/versions/iotlabv1Sliver.py similarity index 77% rename from sfa/rspecs/elements/versions/slabv1Sliver.py rename to sfa/rspecs/elements/versions/iotlabv1Sliver.py index 370f55f3..f26ace6b 100644 --- a/sfa/rspecs/elements/versions/slabv1Sliver.py +++ b/sfa/rspecs/elements/versions/iotlabv1Sliver.py @@ -3,15 +3,15 @@ from sfa.rspecs.elements.sliver import Sliver #from sfa.rspecs.elements.versions.pgv2DiskImage import PGv2DiskImage import sys -class Slabv1Sliver: +class Iotlabv1Sliver: @staticmethod def add_slivers(xml, slivers): if not slivers: - return + return if not isinstance(slivers, list): slivers = [slivers] - for sliver in slivers: + for sliver in slivers: #sliver_elem = xml.add_element('sliver_type') sliver_elem = xml.add_element('sliver') if sliver.get('type'): @@ -20,12 +20,12 @@ class Slabv1Sliver: sliver_elem.set('client_id', sliver['client_id']) #images = sliver.get('disk_images') #if images and isinstance(images, list): - #Slabv1DiskImage.add_images(sliver_elem, images) - Slabv1Sliver.add_sliver_attributes(sliver_elem, sliver.get('tags', [])) - + #Iotlabv1DiskImage.add_images(sliver_elem, images) + Iotlabv1Sliver.add_sliver_attributes(sliver_elem, sliver.get('tags', [])) + @staticmethod def add_sliver_attributes(xml, attributes): - if attributes: + if attributes: for attribute in attributes: if attribute['name'] == 'initscript': xml.add_element('{%s}initscript' % xml.namespaces['planetlab'], name=attribute['value']) @@ -33,26 +33,26 @@ class Slabv1Sliver: attrib_elem = xml.add_element('{%s}info' % self.namespaces['flack']) attrib_dict = eval(tag['value']) for (key, value) in attrib_dict.items(): - attrib_elem.set(key, value) + attrib_elem.set(key, value) @staticmethod def get_slivers(xml, filter={}): xpath = './default:sliver | ./sliver' - + sliver_elems = xml.xpath(xpath) slivers = [] - for sliver_elem in sliver_elems: + for sliver_elem in sliver_elems: sliver = Sliver(sliver_elem.attrib,sliver_elem) - if 'component_id' in xml.attrib: + if 'component_id' in xml.attrib: sliver['component_id'] = xml.attrib['component_id'] if 'name' in sliver_elem.attrib: sliver['type'] = sliver_elem.attrib['name'] - #sliver['images'] = Slabv1DiskImage.get_images(sliver_elem) - + #sliver['images'] = Iotlabv1DiskImage.get_images(sliver_elem) + print>>sys.stderr, "\r\n \r\n SLABV1SLIVER.PY \t\t\t get_slivers sliver %s " %( sliver) slivers.append(sliver) return slivers @staticmethod def get_sliver_attributes(xml, filter={}): - return [] \ No newline at end of file + return [] \ No newline at end of file diff --git a/sfa/rspecs/elements/versions/nitosv1Channel.py b/sfa/rspecs/elements/versions/nitosv1Channel.py index 1ebf7ab6..60582e35 100644 --- a/sfa/rspecs/elements/versions/nitosv1Channel.py +++ b/sfa/rspecs/elements/versions/nitosv1Channel.py @@ -48,7 +48,7 @@ class NITOSv1Channel: channel_elems = [] for channel in channels: - channel_fields = ['channel_num', 'frequency', 'standard'] + channel_fields = ['channel_num', 'frequency', 'standard', 'component_id'] channel_elem = spectrum_elem.add_instance('channel', channel, channel_fields) channel_elems.append(channel_elem) @@ -67,6 +67,7 @@ class NITOSv1Channel: channel['channel_num'] = channel_elem.attrib['channel_num'] channel['frequency'] = channel_elem.attrib['frequency'] channel['standard'] = channel_elem.attrib['standard'] + channel['component_id'] = channel_elem.attrib['component_id'] channels.append(channel) return channels diff --git a/sfa/rspecs/elements/versions/nitosv1Lease.py b/sfa/rspecs/elements/versions/nitosv1Lease.py index f5334a74..99e815a5 100644 --- a/sfa/rspecs/elements/versions/nitosv1Lease.py +++ b/sfa/rspecs/elements/versions/nitosv1Lease.py @@ -54,7 +54,7 @@ class NITOSv1Lease: lease_elems = [] for lease in grouped_leases: - lease[0]['start_time'] = datetime_to_string(utcparse(lease[0]['start_time'])) + #lease[0]['start_time'] = datetime_to_string(utcparse(lease[0]['start_time'])) lease_fields = ['slice_id', 'start_time', 'duration'] lease_elem = network_elem.add_instance('lease', lease[0], lease_fields) @@ -67,9 +67,9 @@ class NITOSv1Lease: # add reserved channels of this lease #channels = [{'channel_id': 1}, {'channel_id': 2}] for channel in channels: - channel['start_time'] = datetime_to_string(utcparse(channel['start_time'])) + #channel['start_time'] = datetime_to_string(utcparse(channel['start_time'])) if channel['slice_id'] == lease[0]['slice_id'] and channel['start_time'] == lease[0]['start_time'] and channel['duration'] == lease[0]['duration']: - lease_elem.add_instance('channel', channel, ['channel_num']) + lease_elem.add_instance('channel', channel, ['component_id']) @staticmethod @@ -88,19 +88,23 @@ class NITOSv1Lease: for node_elem in node_elems: lease = Lease(lease_elem.attrib, lease_elem) lease['slice_id'] = lease_elem.attrib['slice_id'] - lease['start_time'] = datetime_to_epoch(utcparse(lease_elem.attrib['start_time'])) + #lease['start_time'] = datetime_to_epoch(utcparse(lease_elem.attrib['start_time'])) + lease['start_time'] = lease_elem.attrib['start_time'] lease['duration'] = lease_elem.attrib['duration'] lease['component_id'] = node_elem.attrib['component_id'] + lease['type'] = 'node' leases.append(lease) #get channels channel_elems = lease_elem.xpath('./default:channel | ./channel') for channel_elem in channel_elems: channel = Channel(channel_elem.attrib, channel_elem) channel['slice_id'] = lease_elem.attrib['slice_id'] - channel['start_time'] = datetime_to_epoch(utcparse(lease_elem.attrib['start_time'])) + #channel['start_time'] = datetime_to_epoch(utcparse(lease_elem.attrib['start_time'])) + channel['start_time'] = lease_elem.attrib['start_time'] channel['duration'] = lease_elem.attrib['duration'] - channel['channel_num'] = channel_elem.attrib['channel_num'] + channel['component_id'] = channel_elem.attrib['component_id'] + channel['type'] = 'channel' channels.append(channel) - return (leases, channels) + return leases + channels diff --git a/sfa/rspecs/elements/versions/nitosv1Node.py b/sfa/rspecs/elements/versions/nitosv1Node.py index 90150777..44b9b523 100644 --- a/sfa/rspecs/elements/versions/nitosv1Node.py +++ b/sfa/rspecs/elements/versions/nitosv1Node.py @@ -20,7 +20,7 @@ from sfa.rspecs.elements.versions.pgv2Services import PGv2Services class NITOSv1Node: @staticmethod - def add_nodes(xml, nodes): + def add_nodes(xml, nodes, rspec_content_type=None): network_elems = xml.xpath('//network') if len(network_elems) > 0: network_elem = network_elems[0] @@ -102,6 +102,10 @@ class NITOSv1Node: tag_elem = node_elem.add_element(tag['tagname']) tag_elem.set_text(tag['value']) NITOSv1Sliver.add_slivers(node_elem, node.get('slivers', [])) + + # add sliver tag in Request Rspec + if rspec_content_type == "request": + node_elem.add_instance('sliver', '', []) @staticmethod def add_slivers(xml, slivers): diff --git a/sfa/rspecs/elements/versions/nitosv1Sliver.py b/sfa/rspecs/elements/versions/nitosv1Sliver.py index 3fe074b2..0f40211d 100644 --- a/sfa/rspecs/elements/versions/nitosv1Sliver.py +++ b/sfa/rspecs/elements/versions/nitosv1Sliver.py @@ -5,7 +5,7 @@ from sfa.rspecs.elements.element import Element from sfa.rspecs.elements.sliver import Sliver from sfa.rspecs.elements.versions.nitosv1PLTag import NITOSv1PLTag -from sfa.planetlab.plxrn import PlXrn +#from sfa.planetlab.plxrn import PlXrn class NITOSv1Sliver: @@ -22,7 +22,7 @@ class NITOSv1Sliver: for tag in tags: NITOSv1Sliver.add_sliver_attribute(sliver_elem, tag['tagname'], tag['value']) if sliver.get('sliver_id'): - name = PlXrn(xrn=sliver.get('sliver_id')).pl_slicename() + name = Xrn(xrn=sliver.get('sliver_id')).get_hrn().split('.')[-1] sliver_elem.set('name', name) @staticmethod diff --git a/sfa/rspecs/elements/versions/pgv2Lease.py b/sfa/rspecs/elements/versions/pgv2Lease.py index 1d08d5a1..68c44f0c 100644 --- a/sfa/rspecs/elements/versions/pgv2Lease.py +++ b/sfa/rspecs/elements/versions/pgv2Lease.py @@ -15,7 +15,6 @@ from sfa.rspecs.elements.versions.pgv2SliverType import PGv2SliverType from sfa.rspecs.elements.versions.pgv2Interface import PGv2Interface from sfa.rspecs.elements.lease import Lease -from sfa.planetlab.plxrn import xrn_to_hostname class PGv2Lease: @staticmethod @@ -40,7 +39,7 @@ class PGv2Lease: lease_elems = [] for lease in grouped_leases: - lease[0]['start_time'] = datetime_to_string(utcparse(lease[0]['start_time'])) + #lease[0]['start_time'] = datetime_to_string(utcparse(lease[0]['start_time'])) lease_fields = ['slice_id', 'start_time', 'duration'] lease_elem = xml.add_instance('lease', lease[0], lease_fields) @@ -67,7 +66,8 @@ class PGv2Lease: for node_elem in node_elems: lease = Lease(lease_elem.attrib, lease_elem) lease['slice_id'] = lease_elem.attrib['slice_id'] - lease['start_time'] = datetime_to_epoch(utcparse(lease_elem.attrib['start_time'])) + #lease['start_time'] = datetime_to_epoch(utcparse(lease_elem.attrib['start_time'])) + lease['start_time'] = lease_elem.attrib['start_time'] lease['duration'] = lease_elem.attrib['duration'] lease['component_id'] = node_elem.attrib['component_id'] leases.append(lease) diff --git a/sfa/rspecs/elements/versions/pgv2Node.py b/sfa/rspecs/elements/versions/pgv2Node.py index fb9a9ac5..d553d219 100644 --- a/sfa/rspecs/elements/versions/pgv2Node.py +++ b/sfa/rspecs/elements/versions/pgv2Node.py @@ -1,4 +1,4 @@ -from sfa.util.xrn import Xrn +from sfa.util.xrn import Xrn, get_leaf from sfa.util.xml import XpathFilter from sfa.rspecs.elements.node import NodeElement @@ -12,13 +12,14 @@ from sfa.rspecs.elements.pltag import PLTag from sfa.rspecs.elements.versions.pgv2Services import PGv2Services from sfa.rspecs.elements.versions.pgv2SliverType import PGv2SliverType from sfa.rspecs.elements.versions.pgv2Interface import PGv2Interface +from sfa.rspecs.elements.versions.sfav1PLTag import SFAv1PLTag from sfa.rspecs.elements.granularity import Granularity +from sfa.rspecs.elements.attribute import Attribute -from sfa.planetlab.plxrn import xrn_to_hostname class PGv2Node: @staticmethod - def add_nodes(xml, nodes): + def add_nodes(xml, nodes, rspec_content_type=None): node_elems = [] for node in nodes: node_fields = ['component_manager_id', 'component_id', 'client_id', 'sliver_id', 'exclusive'] @@ -26,7 +27,7 @@ class PGv2Node: node_elems.append(node_elem) # set component name if node.get('component_id'): - component_name = xrn_to_hostname(node['component_id']) + component_name = Xrn.unescape(get_leaf(Xrn(node['component_id']).get_hrn())) node_elem.set('component_name', component_name) # set hardware types if node.get('hardware_types'): @@ -37,7 +38,7 @@ class PGv2Node: node_elem.add_instance('location', node['location'], Location.fields) # set granularity - if node['exclusive'] == "true": + if node.get('exclusive') == "true": granularity = node.get('granularity') node_elem.add_instance('granularity', granularity, granularity.fields) # set interfaces @@ -54,15 +55,32 @@ class PGv2Node: slivers = node.get('slivers', []) if not slivers: # we must still advertise the available sliver types - slivers = Sliver({'type': 'plab-vserver'}) + if node.get('sliver_type'): + slivers = Sliver({'type': node['sliver_type']}) + else: + # Planet lab + slivers = Sliver({'type': 'plab-vserver'}) # we must also advertise the available initscripts slivers['tags'] = [] if node.get('pl_initscripts'): for initscript in node.get('pl_initscripts', []): slivers['tags'].append({'name': 'initscript', 'value': initscript['name']}) PGv2SliverType.add_slivers(node_elem, slivers) + + # advertise the node tags + tags = node.get('tags', []) + if tags: + for tag in tags: + tag['name'] = tag.pop('tagname') + node_elem.add_instance('{%s}attribute' % xml.namespaces['planetlab'], tag, ['name', 'value']) + + # add sliver tag in Request Rspec + #if rspec_content_type == "request": + # node_elem.add_instance('sliver', '', []) + return node_elems + @staticmethod def get_nodes(xml, filter={}): xpath = '//node%s | //default:node%s' % (XpathFilter.xpath(filter), XpathFilter.xpath(filter)) @@ -86,11 +104,11 @@ class PGv2Node: # get hardware types hardware_type_elems = node_elem.xpath('./default:hardware_type | ./hardware_type') - node['hardware_types'] = [hw_type.get_instance(HardwareType) for hw_type in hardware_type_elems] + node['hardware_types'] = [dict(hw_type.get_instance(HardwareType)) for hw_type in hardware_type_elems] # get location location_elems = node_elem.xpath('./default:location | ./location') - locations = [location_elem.get_instance(Location) for location_elem in location_elems] + locations = [dict(location_elem.get_instance(Location)) for location_elem in location_elems] if len(locations) > 0: node['location'] = locations[0] @@ -101,19 +119,45 @@ class PGv2Node: # get interfaces iface_elems = node_elem.xpath('./default:interface | ./interface') - node['interfaces'] = [iface_elem.get_instance(Interface) for iface_elem in iface_elems] + node['interfaces'] = [dict(iface_elem.get_instance(Interface)) for iface_elem in iface_elems] # get services node['services'] = PGv2Services.get_services(node_elem) # get slivers node['slivers'] = PGv2SliverType.get_slivers(node_elem) - available_elems = node_elem.xpath('./default:available | ./available') - if len(available_elems) > 0 and 'name' in available_elems[0].attrib: + + # get boot state + available_elems = node_elem.xpath('./default:available | ./available') + if len(available_elems) > 0 and 'now' in available_elems[0].attrib: if available_elems[0].attrib.get('now', '').lower() == 'true': node['boot_state'] = 'boot' else: node['boot_state'] = 'disabled' + + # get initscripts + try: + node['pl_initscripts'] = [] + initscript_elems = node_elem.xpath('./default:sliver_type/planetlab:initscript | ./sliver_type/initscript') + if len(initscript_elems) > 0: + for initscript_elem in initscript_elems: + if 'name' in initscript_elem.attrib: + node['pl_initscripts'].append(dict(initscript_elem.attrib)) + except: + pass + + # get node tags + try: + tag_elems = node_elem.xpath('./planetlab:attribute | ./attribute') + node['tags'] = [] + if len(tag_elems) > 0: + for tag_elem in tag_elems: + tag = dict(tag_elem.get_instance(Attribute)) + tag['tagname'] = tag.pop('name') + node['tags'].append(tag) + except: + pass + return nodes diff --git a/sfa/rspecs/elements/versions/sfav1Lease.py b/sfa/rspecs/elements/versions/sfav1Lease.py index 00c0499f..039d2d13 100644 --- a/sfa/rspecs/elements/versions/sfav1Lease.py +++ b/sfa/rspecs/elements/versions/sfav1Lease.py @@ -17,7 +17,6 @@ from sfa.rspecs.elements.versions.sfav1PLTag import SFAv1PLTag from sfa.rspecs.elements.versions.pgv2Services import PGv2Services from sfa.rspecs.elements.lease import Lease -from sfa.planetlab.plxrn import xrn_to_hostname class SFAv1Lease: @@ -53,7 +52,7 @@ class SFAv1Lease: lease_elems = [] for lease in grouped_leases: - lease[0]['start_time'] = datetime_to_string(utcparse(lease[0]['start_time'])) + #lease[0]['start_time'] = datetime_to_string(utcparse(lease[0]['start_time'])) lease_fields = ['slice_id', 'start_time', 'duration'] lease_elem = network_elem.add_instance('lease', lease[0], lease_fields) @@ -87,7 +86,8 @@ class SFAv1Lease: for node_elem in node_elems: lease = Lease(lease_elem.attrib, lease_elem) lease['slice_id'] = lease_elem.attrib['slice_id'] - lease['start_time'] = datetime_to_epoch(utcparse(lease_elem.attrib['start_time'])) + #lease['start_time'] = datetime_to_epoch(utcparse(lease_elem.attrib['start_time'])) + lease['start_time'] = lease_elem.attrib['start_time'] lease['duration'] = lease_elem.attrib['duration'] lease['component_id'] = node_elem.attrib['component_id'] leases.append(lease) diff --git a/sfa/rspecs/elements/versions/sfav1Node.py b/sfa/rspecs/elements/versions/sfav1Node.py index 997f395a..51076985 100644 --- a/sfa/rspecs/elements/versions/sfav1Node.py +++ b/sfa/rspecs/elements/versions/sfav1Node.py @@ -1,6 +1,6 @@ from sfa.util.sfalogging import logger from sfa.util.xml import XpathFilter -from sfa.util.xrn import Xrn +from sfa.util.xrn import Xrn, get_leaf from sfa.rspecs.elements.element import Element from sfa.rspecs.elements.node import NodeElement @@ -15,12 +15,11 @@ from sfa.rspecs.elements.versions.sfav1Sliver import SFAv1Sliver from sfa.rspecs.elements.versions.sfav1PLTag import SFAv1PLTag from sfa.rspecs.elements.versions.pgv2Services import PGv2Services -from sfa.planetlab.plxrn import xrn_to_hostname class SFAv1Node: @staticmethod - def add_nodes(xml, nodes): + def add_nodes(xml, nodes, rspec_content_type=None): network_elems = xml.xpath('//network') if len(network_elems) > 0: network_elem = network_elems[0] @@ -43,7 +42,7 @@ class SFAv1Node: # set component_name attribute and hostname element if 'component_id' in node and node['component_id']: - component_name = xrn_to_hostname(node['component_id']) + component_name = Xrn.unescape(get_leaf(Xrn(node['component_id']).get_hrn())) node_elem.set('component_name', component_name) hostname_elem = node_elem.add_element('hostname') hostname_elem.set_text(component_name) @@ -94,6 +93,10 @@ class SFAv1Node: tag_elem.set_text(tag['value']) SFAv1Sliver.add_slivers(node_elem, node.get('slivers', [])) + # add sliver tag in Request Rspec + if rspec_content_type == "request": + node_elem.add_instance('sliver', '', []) + @staticmethod def add_slivers(xml, slivers): component_ids = [] @@ -143,7 +146,7 @@ class SFAv1Node: node['authority_id'] = node_elem.attrib['site_id'] # get location location_elems = node_elem.xpath('./default:location | ./location') - locations = [loc_elem.get_instance(Location) for loc_elem in location_elems] + locations = [dict(loc_elem.get_instance(Location)) for loc_elem in location_elems] if len(locations) > 0: node['location'] = locations[0] # get bwlimit @@ -153,7 +156,7 @@ class SFAv1Node: node['bwlimit'] = bwlimits[0] # get interfaces iface_elems = node_elem.xpath('./default:interface | ./interface') - ifaces = [iface_elem.get_instance(Interface) for iface_elem in iface_elems] + ifaces = [dict(iface_elem.get_instance(Interface)) for iface_elem in iface_elems] node['interfaces'] = ifaces # get services node['services'] = PGv2Services.get_services(node_elem) @@ -163,7 +166,7 @@ class SFAv1Node: node['tags'] = SFAv1PLTag.get_pl_tags(node_elem, ignore=NodeElement.fields+["hardware_type"]) # get hardware types hardware_type_elems = node_elem.xpath('./default:hardware_type | ./hardware_type') - node['hardware_types'] = [hw_type.get_instance(HardwareType) for hw_type in hardware_type_elems] + node['hardware_types'] = [dict(hw_type.get_instance(HardwareType)) for hw_type in hardware_type_elems] # temporary... play nice with old slice manager rspec if not node['component_name']: diff --git a/sfa/rspecs/elements/versions/sfav1PLTag.py b/sfa/rspecs/elements/versions/sfav1PLTag.py index a17d5025..b523124f 100644 --- a/sfa/rspecs/elements/versions/sfav1PLTag.py +++ b/sfa/rspecs/elements/versions/sfav1PLTag.py @@ -14,6 +14,6 @@ class SFAv1PLTag: for elem in xml.iterchildren(): if elem.tag not in ignore: pl_tag = PLTag({'tagname': elem.tag, 'value': elem.text}) - pl_tags.append(pl_tag) + pl_tags.append(dict(pl_tag)) return pl_tags diff --git a/sfa/rspecs/elements/versions/sfav1Sliver.py b/sfa/rspecs/elements/versions/sfav1Sliver.py index 851d11c7..a2b07a13 100644 --- a/sfa/rspecs/elements/versions/sfav1Sliver.py +++ b/sfa/rspecs/elements/versions/sfav1Sliver.py @@ -5,7 +5,6 @@ from sfa.rspecs.elements.element import Element from sfa.rspecs.elements.sliver import Sliver from sfa.rspecs.elements.versions.sfav1PLTag import SFAv1PLTag -from sfa.planetlab.plxrn import PlXrn class SFAv1Sliver: diff --git a/sfa/rspecs/versions/slabv1.py b/sfa/rspecs/versions/iotlabv1.py similarity index 82% rename from sfa/rspecs/versions/slabv1.py rename to sfa/rspecs/versions/iotlabv1.py index 39adddac..9d8045f7 100644 --- a/sfa/rspecs/versions/slabv1.py +++ b/sfa/rspecs/versions/iotlabv1.py @@ -3,18 +3,23 @@ from copy import deepcopy from sfa.rspecs.version import RSpecVersion import sys -from sfa.rspecs.elements.versions.slabv1Lease import Slabv1Lease -from sfa.rspecs.elements.versions.slabv1Node import Slabv1Node -from sfa.rspecs.elements.versions.slabv1Sliver import Slabv1Sliver +# from sfa.rspecs.elements.versions.iotlabv1Lease import Iotlabv1Lease +from sfa.rspecs.elements.versions.iotlabv1Node import Iotlabv1Node +from sfa.rspecs.elements.versions.iotlabv1Sliver import Iotlabv1Sliver from sfa.rspecs.elements.versions.sfav1Lease import SFAv1Lease from sfa.util.sfalogging import logger - -class Slabv1(RSpecVersion): + + +class Iotlabv1(RSpecVersion): + """ + Defines Iotlab style RSpec and associated methods to parse and create a + valid Iotlab XML Rspec. + """ #enabled = True - type = 'Slab' + type = 'Iotlab' content_type = 'ad' version = '1' #template = '' % type @@ -27,17 +32,16 @@ class Slabv1(RSpecVersion): } namespaces = dict(extensions.items() + [('default', namespace)]) elements = [] - - # Network + + # Network def get_networks(self): - #WARNING Added //default:network to the xpath + #WARNING Added //default:network to the xpath #otherwise network element not detected 16/07/12 SA - - network_elems = self.xml.xpath('//network | //default:network') + + network_elems = self.xml.xpath('//network | //default:network') networks = [network_elem.get_instance(fields=['name', 'slice']) for \ network_elem in network_elems] - return networks - + return networks def add_network(self, network): network_tags = self.xml.xpath('//network[@name="%s"]' % network) @@ -47,24 +51,22 @@ class Slabv1(RSpecVersion): network_tag = network_tags[0] return network_tag - # Nodes - def get_nodes(self, filter=None): - return Slabv1Node.get_nodes(self.xml, filter) + return Iotlabv1Node.get_nodes(self.xml, filter) def get_nodes_with_slivers(self): - return Slabv1Node.get_nodes_with_slivers(self.xml) - + return Iotlabv1Node.get_nodes_with_slivers(self.xml) + def get_slice_timeslot(self ): - return Slabv1Timeslot.get_slice_timeslot(self.xml) - + return Iotlabv1Timeslot.get_slice_timeslot(self.xml) + def add_connection_information(self, ldap_username, sites_set): - return Slabv1Node.add_connection_information(self.xml,ldap_username, sites_set) - - def add_nodes(self, nodes, check_for_dupes=False): - return Slabv1Node.add_nodes(self.xml,nodes ) - + return Iotlabv1Node.add_connection_information(self.xml,ldap_username, sites_set) + + def add_nodes(self, nodes, check_for_dupes=False, rspec_content_type=None): + return Iotlabv1Node.add_nodes(self.xml,nodes, rspec_content_type) + def merge_node(self, source_node_tag, network, no_dupes = False): logger.debug("SLABV1 merge_node") #if no_dupes and self.get_node_element(node['hostname']): @@ -74,19 +76,19 @@ class Slabv1(RSpecVersion): network_tag.append(deepcopy(source_node_tag)) # Slivers - - def get_sliver_attributes(self, hostname, node, network=None): + + def get_sliver_attributes(self, hostname, node, network=None): print>>sys.stderr, "\r\n \r\n \r\n \t\t SLABV1.PY get_sliver_attributes hostname %s " %(hostname) nodes = self.get_nodes({'component_id': '*%s*' %hostname}) - attribs = [] + attribs = [] print>>sys.stderr, "\r\n \r\n \r\n \t\t SLABV1.PY get_sliver_attributes-----------------nodes %s " %(nodes) if nodes is not None and isinstance(nodes, list) and len(nodes) > 0: node = nodes[0] - #if node : + #if node : #sliver = node.xpath('./default:sliver | ./sliver') #sliver = node.xpath('./default:sliver', namespaces=self.namespaces) sliver = node['slivers'] - + if sliver is not None and isinstance(sliver, list) and len(sliver) > 0: sliver = sliver[0] attribs = sliver @@ -95,7 +97,7 @@ class Slabv1(RSpecVersion): return attribs def get_slice_attributes(self, network=None): - + slice_attributes = [] nodes_with_slivers = self.get_nodes_with_slivers() @@ -120,7 +122,7 @@ class Slabv1(RSpecVersion): value = text attribute = {'name': 'initscript', 'value': value, 'node_id': node} slice_attributes.append(attribute) - + return slice_attributes @@ -140,35 +142,35 @@ class Slabv1(RSpecVersion): def add_slivers(self, hostnames, attributes=[], sliver_urn=None, append=False): # all nodes hould already be present in the rspec. Remove all # nodes that done have slivers - print>>sys.stderr, "\r\n \r\n \r\n \t\t\t SLABv1.PY add_slivers ----->get_node " + print>>sys.stderr, "\r\n \r\n \r\n \t\t\t Iotlabv1.PY add_slivers ----->get_node " for hostname in hostnames: node_elems = self.get_nodes({'component_id': '*%s*' % hostname}) if not node_elems: continue node_elem = node_elems[0] - + # determine sliver types for this node #TODO : add_slivers valid type of sliver needs to be changed 13/07/12 SA - valid_sliver_types = ['slab-node', 'emulab-openvz', 'raw-pc', 'plab-vserver', 'plab-vnode'] + valid_sliver_types = ['iotlab-node', 'emulab-openvz', 'raw-pc', 'plab-vserver', 'plab-vnode'] #valid_sliver_types = ['emulab-openvz', 'raw-pc', 'plab-vserver', 'plab-vnode'] requested_sliver_type = None for sliver_type in node_elem.get('slivers', []): if sliver_type.get('type') in valid_sliver_types: requested_sliver_type = sliver_type['type'] - + if not requested_sliver_type: continue sliver = {'type': requested_sliver_type, 'pl_tags': attributes} - print>>sys.stderr, "\r\n \r\n \r\n \t\t\t SLABv1.PY add_slivers node_elem %s sliver_type %s \r\n \r\n " %(node_elem, sliver_type) + print>>sys.stderr, "\r\n \r\n \r\n \t\t\t Iotlabv1.PY add_slivers node_elem %s sliver_type %s \r\n \r\n " %(node_elem, sliver_type) # remove available element for available_elem in node_elem.xpath('./default:available | ./available'): node_elem.remove(available_elem) - + # remove interface elements for interface_elem in node_elem.xpath('./default:interface | ./interface'): node_elem.remove(interface_elem) - + # remove existing sliver_type elements for sliver_type in node_elem.get('slivers', []): node_elem.element.remove(sliver_type.element) @@ -185,8 +187,8 @@ class Slabv1(RSpecVersion): #node_elem.set('sliver_id', sliver_id) # add the sliver type elemnt - Slabv1Sliver.add_slivers(node_elem.element, sliver) - #Slabv1SliverType.add_slivers(node_elem.element, sliver) + Iotlabv1Sliver.add_slivers(node_elem.element, sliver) + #Iotlabv1SliverType.add_slivers(node_elem.element, sliver) # remove all nodes without slivers if not append: @@ -196,11 +198,11 @@ class Slabv1(RSpecVersion): parent.remove(node_elem.element) def remove_slivers(self, slivers, network=None, no_dupes=False): - Slabv1Node.remove_slivers(self.xml, slivers) - - + Iotlabv1Node.remove_slivers(self.xml, slivers) + + # Utility - + def merge(self, in_rspec): """ Merge contents for specified rspec with current rspec @@ -208,9 +210,9 @@ class Slabv1(RSpecVersion): if not in_rspec: return - + from sfa.rspecs.rspec import RSpec - + if isinstance(in_rspec, RSpec): rspec = in_rspec else: @@ -221,7 +223,7 @@ class Slabv1(RSpecVersion): rspec = RSpec(in_rspec) # just copy over all networks #Attention special get_networks using //default:network xpath - current_networks = self.get_networks() + current_networks = self.get_networks() networks = rspec.version.get_networks() for network in networks: current_network = network.get('name') @@ -232,39 +234,61 @@ class Slabv1(RSpecVersion): - + # Leases def get_leases(self, lease_filter=None): return SFAv1Lease.get_leases(self.xml, lease_filter) - #return Slabv1Lease.get_leases(self.xml, lease_filter) + #return Iotlabv1Lease.get_leases(self.xml, lease_filter) def add_leases(self, leases, network = None, no_dupes=False): SFAv1Lease.add_leases(self.xml, leases) - #Slabv1Lease.add_leases(self.xml, leases) + #Iotlabv1Lease.add_leases(self.xml, leases) + + # Spectrum + + def get_channels(self, filter=None): + return [] + + def add_channels(self, channels, network = None, no_dupes=False): + pass + + # Links + + def get_links(self, network=None): + return [] + + def get_link_requests(self): + return [] + + def add_links(self, links): + pass + def add_link_requests(self, links): + pass + def cleanup(self): # remove unncecessary elements, attributes if self.type in ['request', 'manifest']: # remove 'available' element from remaining node elements self.xml.remove_element('//default:available | //available') - - -class Slabv1Ad(Slabv1): + + +class Iotlabv1Ad(Iotlabv1): enabled = True content_type = 'ad' schema = 'http://senslab.info/resources/rspec/1/ad.xsd' #http://www.geni.net/resources/rspec/3/ad.xsd' template = '' -class Slabv1Request(Slabv1): +class Iotlabv1Request(Iotlabv1): enabled = True content_type = 'request' schema = 'http://senslab.info/resources/rspec/1/request.xsd' #http://www.geni.net/resources/rspec/3/request.xsd template = '' -class Slabv1Manifest(Slabv1): +class Iotlabv1Manifest(Iotlabv1): enabled = True content_type = 'manifest' schema = 'http://senslab.info/resources/rspec/1/manifest.xsd' @@ -275,7 +299,7 @@ class Slabv1Manifest(Slabv1): if __name__ == '__main__': from sfa.rspecs.rspec import RSpec from sfa.rspecs.rspec_elements import * - r = RSpec('/tmp/slab.rspec') - r.load_rspec_elements(Slabv1.elements) - r.namespaces = Slabv1.namespaces + r = RSpec('/tmp/iotlab.rspec') + r.load_rspec_elements(Iotlabv1.elements) + r.namespaces = Iotlabv1.namespaces print r.get(RSpecElements.NODE) diff --git a/sfa/rspecs/versions/nitosv1.py b/sfa/rspecs/versions/nitosv1.py index 29d095ae..af60d8ed 100644 --- a/sfa/rspecs/versions/nitosv1.py +++ b/sfa/rspecs/versions/nitosv1.py @@ -47,8 +47,8 @@ class NITOSv1(RSpecVersion): def get_nodes_with_slivers(self): return NITOSv1Node.get_nodes_with_slivers(self.xml) - def add_nodes(self, nodes, network = None, no_dupes=False): - NITOSv1Node.add_nodes(self.xml, nodes) + def add_nodes(self, nodes, network = None, no_dupes=False, rspec_content_type=None): + NITOSv1Node.add_nodes(self.xml, nodes, rspec_content_type) def merge_node(self, source_node_tag, network, no_dupes=False): if no_dupes and self.get_node_element(node['hostname']): @@ -173,21 +173,15 @@ class NITOSv1(RSpecVersion): # Links def get_links(self, network=None): - return PGv2Link.get_links(self.xml) + return [] def get_link_requests(self): - return PGv2Link.get_link_requests(self.xml) + return [] def add_links(self, links): - networks = self.get_networks() - if len(networks) > 0: - xml = networks[0].element - else: - xml = self.xml - PGv2Link.add_links(xml, links) - + pass def add_link_requests(self, links): - PGv2Link.add_link_requests(self.xml, links) + pass # utility @@ -223,7 +217,8 @@ class NITOSv1(RSpecVersion): def get_leases(self, filter=None): return NITOSv1Lease.get_leases(self.xml, filter) - def add_leases(self, leases, channels, network = None, no_dupes=False): + def add_leases(self, leases_channels, network = None, no_dupes=False): + leases, channels = leases_channels NITOSv1Lease.add_leases(self.xml, leases, channels) # Spectrum diff --git a/sfa/rspecs/versions/pgv2.py b/sfa/rspecs/versions/pgv2.py index fbd8a394..80febc90 100644 --- a/sfa/rspecs/versions/pgv2.py +++ b/sfa/rspecs/versions/pgv2.py @@ -45,8 +45,8 @@ class PGv2(RSpecVersion): def get_nodes_with_slivers(self): return PGv2Node.get_nodes_with_slivers(self.xml) - def add_nodes(self, nodes, check_for_dupes=False): - return PGv2Node.add_nodes(self.xml, nodes) + def add_nodes(self, nodes, check_for_dupes=False, rspec_content_type=None): + return PGv2Node.add_nodes(self.xml, nodes, rspec_content_type) def merge_node(self, source_node_tag): # this is untested @@ -182,6 +182,14 @@ class PGv2(RSpecVersion): def add_leases(self, leases, network = None, no_dupes=False): PGv2Lease.add_leases(self.xml, leases) + # Spectrum + + def get_channels(self, filter=None): + return [] + + def add_channels(self, channels, network = None, no_dupes=False): + pass + # Utility def merge(self, in_rspec): diff --git a/sfa/rspecs/versions/sfav1.py b/sfa/rspecs/versions/sfav1.py index bdfeb47d..4ee8a4c8 100644 --- a/sfa/rspecs/versions/sfav1.py +++ b/sfa/rspecs/versions/sfav1.py @@ -46,8 +46,8 @@ class SFAv1(RSpecVersion): def get_nodes_with_slivers(self): return SFAv1Node.get_nodes_with_slivers(self.xml) - def add_nodes(self, nodes, network = None, no_dupes=False): - SFAv1Node.add_nodes(self.xml, nodes) + def add_nodes(self, nodes, network = None, no_dupes=False, rspec_content_type=None): + SFAv1Node.add_nodes(self.xml, nodes, rspec_content_type) def merge_node(self, source_node_tag, network, no_dupes=False): if no_dupes and self.get_node_element(node['hostname']): @@ -225,6 +225,14 @@ class SFAv1(RSpecVersion): def add_leases(self, leases, network = None, no_dupes=False): SFAv1Lease.add_leases(self.xml, leases) + # Spectrum + + def get_channels(self, filter=None): + return [] + + def add_channels(self, channels, network = None, no_dupes=False): + pass + if __name__ == '__main__': from sfa.rspecs.rspec import RSpec from sfa.rspecs.rspec_elements import * diff --git a/sfa/senslab/OARrestapi.py b/sfa/senslab/OARrestapi.py deleted file mode 100644 index bae1bff6..00000000 --- a/sfa/senslab/OARrestapi.py +++ /dev/null @@ -1,646 +0,0 @@ -#import sys -from httplib import HTTPConnection, HTTPException, NotConnected -import json -#import datetime -#from time import gmtime, strftime -import os.path -import sys -#import urllib -#import urllib2 -from sfa.util.config import Config -#from sfa.util.xrn import hrn_to_urn, get_authority, Xrn, get_leaf - -from sfa.util.sfalogging import logger - - -OAR_REQUEST_POST_URI_DICT = {'POST_job':{'uri': '/oarapi/jobs.json'}, - 'DELETE_jobs_id':{'uri':'/oarapi/jobs/id.json'}, - } - -POST_FORMAT = {'json' : {'content':"application/json", 'object':json},} - -#OARpostdatareqfields = {'resource' :"/nodes=", 'command':"sleep", \ - #'workdir':"/home/", 'walltime':""} - - - -class JsonPage: - """Class used to manipulate jsopn pages given by OAR.""" - def __init__(self): - #All are boolean variables - self.concatenate = False - #Indicates end of data, no more pages to be loaded. - self.end = False - self.next_page = False - #Next query address - self.next_offset = None - #Json page - self.raw_json = None - - def FindNextPage(self): - """ Gets next data page from OAR when the query's results - are too big to be transmitted in a single page. - Uses the "links' item in the json returned to check if - an additionnal page has to be loaded. - Returns : next page , next offset query - """ - if "links" in self.raw_json: - for page in self.raw_json['links']: - if page['rel'] == 'next': - self.concatenate = True - self.next_page = True - self.next_offset = "?" + page['href'].split("?")[1] - print>>sys.stderr, "\r\n \t FindNextPage NEXT LINK" - return - - if self.concatenate : - self.end = True - self.next_page = False - self.next_offset = None - - return - - #Otherwise, no next page and no concatenate, must be a single page - #Concatenate the single page and get out of here. - else: - self.next_page = False - self.concatenate = True - self.next_offset = None - return - - @staticmethod - def ConcatenateJsonPages(saved_json_list): - #reset items list - - tmp = {} - tmp['items'] = [] - - for page in saved_json_list: - tmp['items'].extend(page['items']) - return tmp - - - def ResetNextPage(self): - self.next_page = True - self.next_offset = None - self.concatenate = False - self.end = False - - -class OARrestapi: - def __init__(self, config_file = '/etc/sfa/oar_config.py'): - self.oarserver = {} - - - self.oarserver['uri'] = None - self.oarserver['postformat'] = 'json' - - try: - execfile(config_file, self.__dict__) - - self.config_file = config_file - # path to configuration data - self.config_path = os.path.dirname(config_file) - - except IOError: - raise IOError, "Could not find or load the configuration file: %s" \ - % config_file - #logger.setLevelDebug() - self.oarserver['ip'] = self.OAR_IP - self.oarserver['port'] = self.OAR_PORT - self.jobstates = ['Terminated', 'Hold', 'Waiting', 'toLaunch', \ - 'toError', 'toAckReservation', 'Launching', \ - 'Finishing', 'Running', 'Suspended', 'Resuming',\ - 'Error'] - - self.parser = OARGETParser(self) - - - def GETRequestToOARRestAPI(self, request, strval=None, next_page=None, username = None ): - self.oarserver['uri'] = \ - OARGETParser.OARrequests_uri_dict[request]['uri'] - #Get job details with username - if 'owner' in OARGETParser.OARrequests_uri_dict[request] and username: - self.oarserver['uri'] += OARGETParser.OARrequests_uri_dict[request]['owner'] + username - headers = {} - data = json.dumps({}) - logger.debug("OARrestapi \tGETRequestToOARRestAPI %s" %(request)) - if strval: - self.oarserver['uri'] = self.oarserver['uri'].\ - replace("id",str(strval)) - - if next_page: - self.oarserver['uri'] += next_page - - if username: - headers['X-REMOTE_IDENT'] = username - - logger.debug("OARrestapi: \t GETRequestToOARRestAPI \ - self.oarserver['uri'] %s strval %s" \ - %(self.oarserver['uri'], strval)) - try : - #seems that it does not work if we don't add this - headers['content-length'] = '0' - - conn = HTTPConnection(self.oarserver['ip'], \ - self.oarserver['port']) - conn.request("GET", self.oarserver['uri'], data, headers) - resp = ( conn.getresponse()).read() - conn.close() - - except HTTPException, error : - logger.log_exc("GET_OAR_SRVR : Problem with OAR server : %s " \ - %(error)) - #raise ServerError("GET_OAR_SRVR : Could not reach OARserver") - try: - js_dict = json.loads(resp) - #print "\r\n \t\t\t js_dict keys" , js_dict.keys(), " \r\n", js_dict - return js_dict - - except ValueError, error: - logger.log_exc("Failed to parse Server Response: %s ERROR %s"\ - %(js_dict, error)) - #raise ServerError("Failed to parse Server Response:" + js) - - - def POSTRequestToOARRestAPI(self, request, datadict, username=None): - """ Used to post a job on OAR , along with data associated - with the job. - - """ - - #first check that all params for are OK - try: - self.oarserver['uri'] = OAR_REQUEST_POST_URI_DICT[request]['uri'] - - except KeyError: - logger.log_exc("OARrestapi \tPOSTRequestToOARRestAPI request not \ - valid") - return - if datadict and 'strval' in datadict: - self.oarserver['uri'] = self.oarserver['uri'].replace("id", \ - str(datadict['strval'])) - del datadict['strval'] - - data = json.dumps(datadict) - headers = {'X-REMOTE_IDENT':username, \ - 'content-type': POST_FORMAT['json']['content'], \ - 'content-length':str(len(data))} - try : - - conn = HTTPConnection(self.oarserver['ip'], \ - self.oarserver['port']) - conn.request("POST", self.oarserver['uri'], data, headers) - resp = (conn.getresponse()).read() - conn.close() - except NotConnected: - logger.log_exc("POSTRequestToOARRestAPI NotConnected ERROR: \ - data %s \r\n \t\n \t\t headers %s uri %s" \ - %(data,headers,self.oarserver['uri'])) - - #raise ServerError("POST_OAR_SRVR : error") - - try: - answer = json.loads(resp) - logger.debug("POSTRequestToOARRestAPI : answer %s" %(answer)) - return answer - - except ValueError, error: - logger.log_exc("Failed to parse Server Response: error %s \ - %s" %(error)) - #raise ServerError("Failed to parse Server Response:" + answer) - - - -def AddOarNodeId(tuplelist, value): - """ Adds Oar internal node id to the nodes attributes """ - - tuplelist.append(('oar_id', int(value))) - - -def AddNodeNetworkAddr(dictnode, value): - #Inserts new key. The value associated is a tuple list - node_id = value - - dictnode[node_id] = [('node_id', node_id),('hostname', node_id) ] - - return node_id - -def AddNodeSite(tuplelist, value): - tuplelist.append(('site', str(value))) - -def AddNodeRadio(tuplelist, value): - tuplelist.append(('radio', str(value))) - - -def AddMobility(tuplelist, value): - if value is 0: - tuplelist.append(('mobile', 'False')) - else : - tuplelist.append(('mobile', 'True')) - -def AddPosX(tuplelist, value): - tuplelist.append(('posx', value)) - -def AddPosY(tuplelist, value): - tuplelist.append(('posy', value)) - -def AddPosZ(tuplelist, value): - tuplelist.append(('posz', value)) - -def AddBootState(tuplelist, value): - tuplelist.append(('boot_state', str(value))) - -#Insert a new node into the dictnode dictionary -def AddNodeId(dictnode, value): - #Inserts new key. The value associated is a tuple list - node_id = int(value) - - dictnode[node_id] = [('node_id', node_id)] - return node_id - -def AddHardwareType(tuplelist, value): - value_list = value.split(':') - tuplelist.append(('archi', value_list[0])) - tuplelist.append(('radio', value_list[1])) - - -class OARGETParser: - resources_fulljson_dict = { - 'network_address' : AddNodeNetworkAddr, - 'site': AddNodeSite, - 'radio': AddNodeRadio, - 'mobile': AddMobility, - 'x': AddPosX, - 'y': AddPosY, - 'z':AddPosZ, - 'archi':AddHardwareType, - 'state':AddBootState, - 'id' : AddOarNodeId, - } - - - def __init__(self, srv) : - self.version_json_dict = { - 'api_version' : None , 'apilib_version' :None,\ - 'api_timezone': None, 'api_timestamp': None, 'oar_version': None ,} - self.config = Config() - self.interface_hrn = self.config.SFA_INTERFACE_HRN - self.timezone_json_dict = { - 'timezone': None, 'api_timestamp': None, } - #self.jobs_json_dict = { - #'total' : None, 'links' : [],\ - #'offset':None , 'items' : [], } - #self.jobs_table_json_dict = self.jobs_json_dict - #self.jobs_details_json_dict = self.jobs_json_dict - self.server = srv - self.node_dictlist = {} - - self.json_page = JsonPage() - - self.site_dict = {} - self.SendRequest("GET_version") - - - - - - def ParseVersion(self) : - #print self.json_page.raw_json - #print >>sys.stderr, self.json_page.raw_json - if 'oar_version' in self.json_page.raw_json : - self.version_json_dict.update(api_version = \ - self.json_page.raw_json['api_version'], - apilib_version = self.json_page.raw_json['apilib_version'], - api_timezone = self.json_page.raw_json['api_timezone'], - api_timestamp = self.json_page.raw_json['api_timestamp'], - oar_version = self.json_page.raw_json['oar_version'] ) - else : - self.version_json_dict.update(api_version = \ - self.json_page.raw_json['api'] , - apilib_version = self.json_page.raw_json['apilib'], - api_timezone = self.json_page.raw_json['api_timezone'], - api_timestamp = self.json_page.raw_json['api_timestamp'], - oar_version = self.json_page.raw_json['oar'] ) - - print self.version_json_dict['apilib_version'] - - - def ParseTimezone(self) : - api_timestamp = self.json_page.raw_json['api_timestamp'] - api_tz = self.json_page.raw_json['timezone'] - return api_timestamp, api_tz - - def ParseJobs(self) : - self.jobs_list = [] - print " ParseJobs " - return self.json_page.raw_json - - def ParseJobsTable(self) : - print "ParseJobsTable" - - def ParseJobsDetails (self): - # currently, this function is not used a lot, - #so i have no idea what be usefull to parse, - #returning the full json. NT - #logger.debug("ParseJobsDetails %s " %(self.json_page.raw_json)) - return self.json_page.raw_json - - - def ParseJobsIds(self): - - job_resources = ['wanted_resources', 'name', 'id', 'start_time', \ - 'state','owner','walltime','message'] - - - job_resources_full = ['launching_directory', 'links', \ - 'resubmit_job_id', 'owner', 'events', 'message', \ - 'scheduled_start', 'id', 'array_id', 'exit_code', \ - 'properties', 'state','array_index', 'walltime', \ - 'type', 'initial_request', 'stop_time', 'project',\ - 'start_time', 'dependencies','api_timestamp','submission_time', \ - 'reservation', 'stdout_file', 'types', 'cpuset_name', \ - 'name', 'wanted_resources','queue','stderr_file','command'] - - - job_info = self.json_page.raw_json - #logger.debug("OARESTAPI ParseJobsIds %s" %(self.json_page.raw_json)) - values = [] - try: - for k in job_resources: - values.append(job_info[k]) - return dict(zip(job_resources, values)) - - except KeyError: - logger.log_exc("ParseJobsIds KeyError ") - - - def ParseJobsIdResources(self): - """ Parses the json produced by the request - /oarapi/jobs/id/resources.json. - Returns a list of oar node ids that are scheduled for the - given job id. - - """ - job_resources = [] - for resource in self.json_page.raw_json['items']: - job_resources.append(resource['id']) - - #logger.debug("OARESTAPI \tParseJobsIdResources %s" %(self.json_page.raw_json)) - return job_resources - - def ParseResources(self) : - """ Parses the json produced by a get_resources request on oar.""" - - #logger.debug("OARESTAPI \tParseResources " ) - #resources are listed inside the 'items' list from the json - self.json_page.raw_json = self.json_page.raw_json['items'] - self.ParseNodes() - - def ParseReservedNodes(self): - """ Returns an array containing the list of the reserved nodes """ - - #resources are listed inside the 'items' list from the json - reservation_list = [] - job = {} - #Parse resources info - for json_element in self.json_page.raw_json['items']: - #In case it is a real reservation (not asap case) - if json_element['scheduled_start']: - job['t_from'] = json_element['scheduled_start'] - job['t_until'] = int(json_element['scheduled_start']) + \ - int(json_element['walltime']) - #Get resources id list for the job - job['resource_ids'] = \ - [ node_dict['id'] for node_dict in json_element['resources']] - else: - job['t_from'] = "As soon as possible" - job['t_until'] = "As soon as possible" - job['resource_ids'] = ["Undefined"] - - - job['state'] = json_element['state'] - job['lease_id'] = json_element['id'] - - - job['user'] = json_element['owner'] - #logger.debug("OARRestapi \tParseReservedNodes job %s" %(job)) - reservation_list.append(job) - #reset dict - job = {} - return reservation_list - - def ParseRunningJobs(self): - """ Gets the list of nodes currently in use from the attributes of the - running jobs. - - """ - logger.debug("OARESTAPI \tParseRunningJobs__________________________ ") - #resources are listed inside the 'items' list from the json - nodes = [] - for job in self.json_page.raw_json['items']: - for node in job['nodes']: - nodes.append(node['network_address']) - return nodes - - - - def ParseDeleteJobs(self): - """ No need to parse anything in this function.A POST - is done to delete the job. - - """ - return - - def ParseResourcesFull(self) : - """ This method is responsible for parsing all the attributes - of all the nodes returned by OAR when issuing a get resources full. - The information from the nodes and the sites are separated. - Updates the node_dictlist so that the dictionnary of the platform's - nodes is available afterwards. - - """ - logger.debug("OARRESTAPI ParseResourcesFull________________________ ") - #print self.json_page.raw_json[1] - #resources are listed inside the 'items' list from the json - if self.version_json_dict['apilib_version'] != "0.2.10" : - self.json_page.raw_json = self.json_page.raw_json['items'] - self.ParseNodes() - self.ParseSites() - return self.node_dictlist - - def ParseResourcesFullSites(self) : - """ UNUSED. Originally used to get information from the sites. - ParseResourcesFull is used instead. - - """ - if self.version_json_dict['apilib_version'] != "0.2.10" : - self.json_page.raw_json = self.json_page.raw_json['items'] - self.ParseNodes() - self.ParseSites() - return self.site_dict - - - - def ParseNodes(self): - """ Parse nodes properties from OAR - Put them into a dictionary with key = node id and value is a dictionary - of the node properties and properties'values. - - """ - node_id = None - keys = self.resources_fulljson_dict.keys() - keys.sort() - - for dictline in self.json_page.raw_json: - node_id = None - # dictionary is empty and/or a new node has to be inserted - node_id = self.resources_fulljson_dict['network_address'](\ - self.node_dictlist, dictline['network_address']) - for k in keys: - if k in dictline: - if k == 'network_address': - continue - - self.resources_fulljson_dict[k](\ - self.node_dictlist[node_id], dictline[k]) - - #The last property has been inserted in the property tuple list, - #reset node_id - #Turn the property tuple list (=dict value) into a dictionary - self.node_dictlist[node_id] = dict(self.node_dictlist[node_id]) - node_id = None - - @staticmethod - def slab_hostname_to_hrn( root_auth, hostname): - return root_auth + '.'+ hostname - - - - def ParseSites(self): - """ Returns a list of dictionnaries containing the sites' attributes.""" - - nodes_per_site = {} - config = Config() - #logger.debug(" OARrestapi.py \tParseSites self.node_dictlist %s"\ - #%(self.node_dictlist)) - # Create a list of nodes per site_id - for node_id in self.node_dictlist: - node = self.node_dictlist[node_id] - - if node['site'] not in nodes_per_site: - nodes_per_site[node['site']] = [] - nodes_per_site[node['site']].append(node['node_id']) - else: - if node['node_id'] not in nodes_per_site[node['site']]: - nodes_per_site[node['site']].append(node['node_id']) - - #Create a site dictionary whose key is site_login_base (name of the site) - # and value is a dictionary of properties, including the list - #of the node_ids - for node_id in self.node_dictlist: - node = self.node_dictlist[node_id] - #node.update({'hrn':self.slab_hostname_to_hrn(self.interface_hrn, \ - #node['site'],node['hostname'])}) - node.update({'hrn':self.slab_hostname_to_hrn(self.interface_hrn, node['hostname'])}) - self.node_dictlist.update({node_id:node}) - - if node['site'] not in self.site_dict: - self.site_dict[node['site']] = { - 'site':node['site'], - 'node_ids':nodes_per_site[node['site']], - 'latitude':"48.83726", - 'longitude':"- 2.10336",'name':config.SFA_REGISTRY_ROOT_AUTH, - 'pcu_ids':[], 'max_slices':None, 'ext_consortium_id':None, - 'max_slivers':None, 'is_public':True, 'peer_site_id': None, - 'abbreviated_name':"senslab", 'address_ids': [], - 'url':"http,//www.senslab.info", 'person_ids':[], - 'site_tag_ids':[], 'enabled': True, 'slice_ids':[], - 'date_created': None, 'peer_id': None } - #if node['site_login_base'] not in self.site_dict.keys(): - #self.site_dict[node['site_login_base']] = {'login_base':node['site_login_base'], - #'node_ids':nodes_per_site[node['site_login_base']], - #'latitude':"48.83726", - #'longitude':"- 2.10336",'name':"senslab", - #'pcu_ids':[], 'max_slices':None, 'ext_consortium_id':None, - #'max_slivers':None, 'is_public':True, 'peer_site_id': None, - #'abbreviated_name':"senslab", 'address_ids': [], - #'url':"http,//www.senslab.info", 'person_ids':[], - #'site_tag_ids':[], 'enabled': True, 'slice_ids':[], - #'date_created': None, 'peer_id': None } - - - - - OARrequests_uri_dict = { - 'GET_version': - {'uri':'/oarapi/version.json', 'parse_func': ParseVersion}, - 'GET_timezone': - {'uri':'/oarapi/timezone.json' ,'parse_func': ParseTimezone }, - 'GET_jobs': - {'uri':'/oarapi/jobs.json','parse_func': ParseJobs}, - 'GET_jobs_id': - {'uri':'/oarapi/jobs/id.json','parse_func': ParseJobsIds}, - 'GET_jobs_id_resources': - {'uri':'/oarapi/jobs/id/resources.json',\ - 'parse_func': ParseJobsIdResources}, - 'GET_jobs_table': - {'uri':'/oarapi/jobs/table.json','parse_func': ParseJobsTable}, - 'GET_jobs_details': - {'uri':'/oarapi/jobs/details.json',\ - 'parse_func': ParseJobsDetails}, - 'GET_reserved_nodes': - {'uri': - '/oarapi/jobs/details.json?state=Running,Waiting,Launching',\ - 'owner':'&user=', - 'parse_func':ParseReservedNodes}, - - - 'GET_running_jobs': - {'uri':'/oarapi/jobs/details.json?state=Running',\ - 'parse_func':ParseRunningJobs}, - 'GET_resources_full': - {'uri':'/oarapi/resources/full.json',\ - 'parse_func': ParseResourcesFull}, - 'GET_sites': - {'uri':'/oarapi/resources/full.json',\ - 'parse_func': ParseResourcesFullSites}, - 'GET_resources': - {'uri':'/oarapi/resources.json' ,'parse_func': ParseResources}, - 'DELETE_jobs_id': - {'uri':'/oarapi/jobs/id.json' ,'parse_func': ParseDeleteJobs} - } - - - - - def SendRequest(self, request, strval = None , username = None): - """ Connects to OAR , sends the valid GET requests and uses - the appropriate json parsing functions. - - """ - save_json = None - - self.json_page.ResetNextPage() - save_json = [] - - if request in self.OARrequests_uri_dict : - while self.json_page.next_page: - self.json_page.raw_json = self.server.GETRequestToOARRestAPI(\ - request, \ - strval, \ - self.json_page.next_offset, \ - username) - self.json_page.FindNextPage() - if self.json_page.concatenate: - save_json.append(self.json_page.raw_json) - - if self.json_page.concatenate and self.json_page.end : - self.json_page.raw_json = \ - self.json_page.ConcatenateJsonPages(save_json) - - return self.OARrequests_uri_dict[request]['parse_func'](self) - else: - logger.error("OARRESTAPI OARGetParse __init__ : ERROR_REQUEST " \ - %(request)) - diff --git a/sfa/senslab/slabaggregate.py b/sfa/senslab/slabaggregate.py deleted file mode 100644 index 7e15d111..00000000 --- a/sfa/senslab/slabaggregate.py +++ /dev/null @@ -1,399 +0,0 @@ -#import time -from sfa.util.xrn import hrn_to_urn, urn_to_hrn, get_authority - -from sfa.rspecs.rspec import RSpec -#from sfa.rspecs.elements.location import Location -from sfa.rspecs.elements.hardware_type import HardwareType -from sfa.rspecs.elements.login import Login -from sfa.rspecs.elements.services import ServicesElement -from sfa.rspecs.elements.sliver import Sliver -from sfa.rspecs.elements.lease import Lease -from sfa.rspecs.elements.granularity import Granularity -from sfa.rspecs.version_manager import VersionManager - - -from sfa.rspecs.elements.versions.slabv1Node import SlabPosition, SlabNode, \ - SlabLocation -from sfa.util.sfalogging import logger - -from sfa.util.xrn import Xrn - -def slab_xrn_to_hostname(xrn): - return Xrn.unescape(Xrn(xrn=xrn, type='node').get_leaf()) - -def slab_xrn_object(root_auth, hostname): - """Attributes are urn and hrn. - Get the hostname using slab_xrn_to_hostname on the urn. - - :return: the senslab node's xrn - :rtype: Xrn - """ - return Xrn('.'.join( [root_auth, Xrn.escape(hostname)]), type='node') - -class SlabAggregate: - - sites = {} - nodes = {} - api = None - interfaces = {} - links = {} - node_tags = {} - - prepared = False - - user_options = {} - - def __init__(self, driver): - self.driver = driver - - def get_slice_and_slivers(self, slice_xrn, login=None): - """ - Get the slices and the associated leases if any from the senslab - testbed. For each slice, get the nodes in the associated lease - and create a sliver with the necessary info and insertinto the sliver - dictionary, keyed on the node hostnames. - Returns a dict of slivers based on the sliver's node_id. - Called by get_rspec. - - - :param slice_xrn: xrn of the slice - :param login: user's login on senslab ldap - - :type slice_xrn: string - :type login: string - :reutnr : a list of slices dict and a dictionary of Sliver object - :rtype: (list, dict) - - ..note: There is no slivers in senslab, only leases. - - """ - slivers = {} - sfa_slice = None - if slice_xrn is None: - return (sfa_slice, slivers) - slice_urn = hrn_to_urn(slice_xrn, 'slice') - slice_hrn, _ = urn_to_hrn(slice_xrn) - slice_name = slice_hrn - - slices = self.driver.slab_api.GetSlices(slice_filter= str(slice_name), \ - slice_filter_type = 'slice_hrn', \ - login=login) - - logger.debug("Slabaggregate api \tget_slice_and_slivers \ - sfa_slice %s \r\n slices %s self.driver.hrn %s" \ - %(sfa_slice, slices, self.driver.hrn)) - if slices == []: - return (sfa_slice, slivers) - - - # sort slivers by node id , if there is a job - #and therefore, node allocated to this slice - for sfa_slice in slices: - try: - node_ids_list = sfa_slice['node_ids'] - except KeyError: - logger.log_exc("SLABAGGREGATE \t \ - get_slice_and_slivers KeyError ") - continue - - for node in node_ids_list: - sliver_xrn = Xrn(slice_urn, type='sliver', id=node) - sliver_xrn.set_authority(self.driver.hrn) - sliver = Sliver({'sliver_id':sliver_xrn.urn, - 'name': sfa_slice['hrn'], - 'type': 'slab-node', - 'tags': []}) - - slivers[node] = sliver - - - #Add default sliver attribute : - #connection information for senslab - if get_authority (sfa_slice['hrn']) == self.driver.slab_api.root_auth: - tmp = sfa_slice['hrn'].split('.') - ldap_username = tmp[1].split('_')[0] - ssh_access = None - slivers['default_sliver'] = {'ssh': ssh_access , \ - 'login': ldap_username} - - #TODO get_slice_and_slivers Find the login of the external user - - logger.debug("SLABAGGREGATE api get_slice_and_slivers slivers %s "\ - %(slivers)) - return (slices, slivers) - - - - def get_nodes(self, slices=None, slivers=[], options=None): - # NT: the semantic of this function is not clear to me : - # if slice is not defined, then all the nodes should be returned - # if slice is defined, we should return only the nodes that - # are part of this slice - # but what is the role of the slivers parameter ? - # So i assume that slice['node_ids'] will be the same as slivers for us - #filter_dict = {} - #if slice_xrn: - #if not slices or not slices['node_ids']: - #return ([],[]) - #tags_filter = {} - - # get the granularity in second for the reservation system - grain = self.driver.slab_api.GetLeaseGranularity() - - - nodes = self.driver.slab_api.GetNodes() - #geni_available = options.get('geni_available') - #if geni_available: - #filter['boot_state'] = 'boot' - - #filter.update({'peer_id': None}) - #nodes = self.driver.slab_api.GetNodes(filter['hostname']) - - #site_ids = [] - #interface_ids = [] - #tag_ids = [] - nodes_dict = {} - - #for node in nodes: - - #nodes_dict[node['node_id']] = node - #logger.debug("SLABAGGREGATE api get_nodes nodes %s "\ - #%(nodes )) - # get sites - #sites_dict = self.get_sites({'site_id': site_ids}) - # get interfaces - #interfaces = self.get_interfaces({'interface_id':interface_ids}) - # get tags - #node_tags = self.get_node_tags(tags_filter) - - #if slices, this means we got to list all the nodes given to this slice - # Make a list of all the nodes in the slice before getting their - #attributes - rspec_nodes = [] - slice_nodes_list = [] - logger.debug("SLABAGGREGATE api get_nodes slice_nodes_list %s "\ - %(slices )) - if slices is not None: - for one_slice in slices: - try: - slice_nodes_list = one_slice['node_ids'] - except KeyError: - pass - #for node in one_slice['node_ids']: - #slice_nodes_list.append(node) - - reserved_nodes = self.driver.slab_api.GetNodesCurrentlyInUse() - logger.debug("SLABAGGREGATE api get_nodes slice_nodes_list %s "\ - %(slice_nodes_list)) - for node in nodes: - nodes_dict[node['node_id']] = node - if slice_nodes_list == [] or node['hostname'] in slice_nodes_list: - - rspec_node = SlabNode() - # xxx how to retrieve site['login_base'] - #site_id=node['site_id'] - #site=sites_dict[site_id] - - rspec_node['mobile'] = node['mobile'] - rspec_node['archi'] = node['archi'] - rspec_node['radio'] = node['radio'] - - slab_xrn = slab_xrn_object(self.driver.slab_api.root_auth, \ - node['hostname']) - rspec_node['component_id'] = slab_xrn.urn - rspec_node['component_name'] = node['hostname'] - rspec_node['component_manager_id'] = \ - hrn_to_urn(self.driver.slab_api.root_auth, \ - 'authority+sa') - - # Senslab's nodes are federated : there is only one authority - # for all Senslab sites, registered in SFA. - # Removing the part including the site - # in authority_id SA 27/07/12 - rspec_node['authority_id'] = rspec_node['component_manager_id'] - - # do not include boot state ( element) - #in the manifest rspec - - - rspec_node['boot_state'] = node['boot_state'] - if node['hostname'] in reserved_nodes: - rspec_node['boot_state'] = "Reserved" - rspec_node['exclusive'] = 'true' - rspec_node['hardware_types'] = [HardwareType({'name': \ - 'slab-node'})] - - - location = SlabLocation({'country':'France', 'site': \ - node['site']}) - rspec_node['location'] = location - - - position = SlabPosition() - for field in position : - try: - position[field] = node[field] - except KeyError, error : - logger.log_exc("SLABAGGREGATE\t get_nodes \ - position %s "%(error)) - - rspec_node['position'] = position - #rspec_node['interfaces'] = [] - - # Granularity - granularity = Granularity({'grain': grain}) - rspec_node['granularity'] = granularity - rspec_node['tags'] = [] - if node['hostname'] in slivers: - # add sliver info - sliver = slivers[node['hostname']] - rspec_node['sliver_id'] = sliver['sliver_id'] - rspec_node['client_id'] = node['hostname'] - rspec_node['slivers'] = [sliver] - - # slivers always provide the ssh service - login = Login({'authentication': 'ssh-keys', \ - 'hostname': node['hostname'], 'port':'22', \ - 'username': sliver['name']}) - service = Services({'login': login}) - rspec_node['services'] = [service] - rspec_nodes.append(rspec_node) - - return (rspec_nodes) - #def get_all_leases(self, slice_record = None): - def get_all_leases(self): - """ - Get list of lease dictionaries which all have the mandatory keys - ('lease_id', 'hostname', 'site_id', 'name', 'start_time', 'duration'). - All the leases running or scheduled are returned. - - - ..note::There is no filtering of leases within a given time frame. - All the running or scheduled leases are returned. options - removed SA 15/05/2013 - - - """ - - #now = int(time.time()) - #lease_filter = {'clip': now } - - #if slice_record: - #lease_filter.update({'name': slice_record['name']}) - - #leases = self.driver.slab_api.GetLeases(lease_filter) - leases = self.driver.slab_api.GetLeases() - grain = self.driver.slab_api.GetLeaseGranularity() - site_ids = [] - rspec_leases = [] - for lease in leases: - #as many leases as there are nodes in the job - for node in lease['reserved_nodes']: - rspec_lease = Lease() - rspec_lease['lease_id'] = lease['lease_id'] - #site = node['site_id'] - slab_xrn = slab_xrn_object(self.driver.slab_api.root_auth, node) - rspec_lease['component_id'] = slab_xrn.urn - #rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn,\ - #site, node['hostname']) - try: - rspec_lease['slice_id'] = lease['slice_id'] - except KeyError: - #No info on the slice used in slab_xp table - pass - rspec_lease['start_time'] = lease['t_from'] - rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) \ - / grain - rspec_leases.append(rspec_lease) - return rspec_leases - - - -#from plc/aggregate.py - def get_rspec(self, slice_xrn=None, login=None, version = None, \ - options=None): - - rspec = None - version_manager = VersionManager() - version = version_manager.get_version(version) - logger.debug("SlabAggregate \t get_rspec ***version %s \ - version.type %s version.version %s options %s \r\n" \ - %(version,version.type,version.version,options)) - - if slice_xrn is None: - rspec_version = version_manager._get_version(version.type, \ - version.version, 'ad') - - else: - rspec_version = version_manager._get_version(version.type, \ - version.version, 'manifest') - - slices, slivers = self.get_slice_and_slivers(slice_xrn, login) - #at this point sliver may be empty if no senslab job - #is running for this user/slice. - rspec = RSpec(version=rspec_version, user_options=options) - - - #if slice and 'expires' in slice: - #rspec.xml.set('expires',\ - #datetime_to_string(utcparse(slice['expires'])) - # add sliver defaults - #nodes, links = self.get_nodes(slice, slivers) - logger.debug("\r\n \r\n SlabAggregate \tget_rspec *** \ - slice_xrn %s slices %s\r\n \r\n"\ - %(slice_xrn, slices)) - - if options is not None: - lease_option = options['list_leases'] - else: - #If no options are specified, at least print the resources - lease_option = 'all' - #if slice_xrn : - #lease_option = 'all' - - - if lease_option in ['all', 'resources']: - #if not options.get('list_leases') or options.get('list_leases') - #and options['list_leases'] != 'leases': - nodes = self.get_nodes(slices, slivers) - logger.debug("\r\n \r\n SlabAggregate \ lease_option %s \ - get rspec ******* nodes %s"\ - %(lease_option, nodes[0])) - - sites_set = set([node['location']['site'] for node in nodes] ) - - #In case creating a job, slice_xrn is not set to None - rspec.version.add_nodes(nodes) - if slice_xrn : - #Get user associated with this slice - #user = dbsession.query(RegRecord).filter_by(record_id = \ - #slices['record_id_user']).first() - - #ldap_username = (user.hrn).split('.')[1] - - - #for one_slice in slices : - ldap_username = slices[0]['hrn'] - tmp = ldap_username.split('.') - ldap_username = tmp[1].split('_')[0] - - if version.type == "Slab": - rspec.version.add_connection_information(ldap_username, \ - sites_set) - - default_sliver = slivers.get('default_sliver', []) - if default_sliver: - #default_sliver_attribs = default_sliver.get('tags', []) - logger.debug("SlabAggregate \tget_rspec **** \ - default_sliver%s \r\n" %(default_sliver)) - for attrib in default_sliver: - rspec.version.add_default_sliver_attribute(attrib, \ - default_sliver[attrib]) - if lease_option in ['all','leases']: - #leases = self.get_all_leases(slices) - leases = self.get_all_leases() - rspec.version.add_leases(leases) - - #logger.debug("SlabAggregate \tget_rspec ******* rspec_toxml %s \r\n"\ - #%(rspec.toxml())) - return rspec.toxml() diff --git a/sfa/senslab/slabdriver.py b/sfa/senslab/slabdriver.py deleted file mode 100644 index 8ab31d76..00000000 --- a/sfa/senslab/slabdriver.py +++ /dev/null @@ -1,770 +0,0 @@ -from sfa.util.faults import SliverDoesNotExist, UnknownSfaType -from sfa.util.sfalogging import logger -from sfa.storage.alchemy import dbsession -from sfa.storage.model import RegRecord - - - -from sfa.managers.driver import Driver -from sfa.rspecs.version_manager import VersionManager -from sfa.rspecs.rspec import RSpec - -from sfa.util.xrn import Xrn, hrn_to_urn, get_authority - - -from sfa.senslab.slabpostgres import SlabDB - - -from sfa.senslab.slabaggregate import SlabAggregate, slab_xrn_to_hostname - -from sfa.senslab.slabslices import SlabSlices - - -from sfa.senslab.slabapi import SlabTestbedAPI - - - -class SlabDriver(Driver): - """ Senslab Driver class inherited from Driver generic class. - - Contains methods compliant with the SFA standard and the testbed - infrastructure (calls to LDAP and OAR). - - .. seealso:: Driver class - - """ - def __init__(self, config): - """ - - Sets the senslab SFA config parameters , - instanciates the testbed api and the senslab database. - - :param config: senslab SFA configuration object - :type config: Config object - """ - Driver.__init__ (self, config) - self.config = config - - self.db = SlabDB(config, debug = False) - self.slab_api = SlabTestbedAPI(config) - self.cache = None - - def augment_records_with_testbed_info (self, record_list ): - """ - - Adds specific testbed info to the records. - - :param record_list: list of sfa dictionaries records - :type record_list: list - :return: list of records with extended information in each record - :rtype: list - """ - return self.fill_record_info (record_list) - - def fill_record_info(self, record_list): - """ - For each SFA record, fill in the senslab specific and SFA specific - fields in the record. - - :param record_list: list of sfa dictionaries records - :type record_list: list - :return: list of records with extended information in each record - :rtype: list - - .. warnings:: Should not be modifying record_list directly because modi - fication are kept outside the method's scope. Howerver, there is no - other way to do it given the way it's called in registry manager. - """ - - logger.debug("SLABDRIVER \tfill_record_info records %s " %(record_list)) - if not isinstance(record_list, list): - record_list = [record_list] - - - try: - for record in record_list: - #If the record is a SFA slice record, then add information - #about the user of this slice. This kind of - #information is in the Senslab's DB. - if str(record['type']) == 'slice': - if 'reg_researchers' in record and \ - isinstance(record['reg_researchers'], list) : - record['reg_researchers'] = \ - record['reg_researchers'][0].__dict__ - record.update({'PI':[record['reg_researchers']['hrn']], - 'researcher': [record['reg_researchers']['hrn']], - 'name':record['hrn'], - 'oar_job_id':[], - 'node_ids': [], - 'person_ids':[record['reg_researchers']['record_id']], - 'geni_urn':'', #For client_helper.py compatibility - 'keys':'', #For client_helper.py compatibility - 'key_ids':''}) #For client_helper.py compatibility - - - #Get slab slice record and oar job id if any. - recslice_list = self.slab_api.GetSlices(slice_filter = \ - str(record['hrn']),\ - slice_filter_type = 'slice_hrn') - - - logger.debug("SLABDRIVER \tfill_record_info \ - TYPE SLICE RECUSER record['hrn'] %s ecord['oar_job_id']\ - %s " %(record['hrn'], record['oar_job_id'])) - del record['reg_researchers'] - try: - for rec in recslice_list: - logger.debug("SLABDRIVER\r\n \t \ - fill_record_info oar_job_id %s " \ - %(rec['oar_job_id'])) - - record['node_ids'] = [ self.slab_api.root_auth + \ - hostname for hostname in rec['node_ids']] - except KeyError: - pass - - - logger.debug( "SLABDRIVER.PY \t fill_record_info SLICE \ - recslice_list %s \r\n \t RECORD %s \r\n \ - \r\n" %(recslice_list, record)) - - if str(record['type']) == 'user': - #The record is a SFA user record. - #Get the information about his slice from Senslab's DB - #and add it to the user record. - recslice_list = self.slab_api.GetSlices(\ - slice_filter = record['record_id'],\ - slice_filter_type = 'record_id_user') - - logger.debug( "SLABDRIVER.PY \t fill_record_info TYPE USER \ - recslice_list %s \r\n \t RECORD %s \r\n" \ - %(recslice_list , record)) - #Append slice record in records list, - #therefore fetches user and slice info again(one more loop) - #Will update PIs and researcher for the slice - - recuser = recslice_list[0]['reg_researchers'] - logger.debug( "SLABDRIVER.PY \t fill_record_info USER \ - recuser %s \r\n \r\n" %(recuser)) - recslice = {} - recslice = recslice_list[0] - recslice.update({'PI':[recuser['hrn']], - 'researcher': [recuser['hrn']], - 'name':record['hrn'], - 'node_ids': [], - 'oar_job_id': [], - 'person_ids':[recuser['record_id']]}) - try: - for rec in recslice_list: - recslice['oar_job_id'].append(rec['oar_job_id']) - except KeyError: - pass - - recslice.update({'type':'slice', \ - 'hrn':recslice_list[0]['hrn']}) - - - #GetPersons takes [] as filters - user_slab = self.slab_api.GetPersons([record]) - - - record.update(user_slab[0]) - #For client_helper.py compatibility - record.update( { 'geni_urn':'', - 'keys':'', - 'key_ids':'' }) - record_list.append(recslice) - - logger.debug("SLABDRIVER.PY \tfill_record_info ADDING SLICE\ - INFO TO USER records %s" %(record_list)) - - - except TypeError, error: - logger.log_exc("SLABDRIVER \t fill_record_info EXCEPTION %s"\ - %(error)) - - return record_list - - - def sliver_status(self, slice_urn, slice_hrn): - """ - Receive a status request for slice named urn/hrn - urn:publicid:IDN+senslab+nturro_slice hrn senslab.nturro_slice - shall return a structure as described in - http://groups.geni.net/geni/wiki/GAPI_AM_API_V2#SliverStatus - NT : not sure if we should implement this or not, but used by sface. - - :param slice_urn: slice urn - :type slice_urn: string - :param slice_hrn: slice hrn - :type slice_hrn: string - - """ - - - #First get the slice with the slice hrn - slice_list = self.slab_api.GetSlices(slice_filter = slice_hrn, \ - slice_filter_type = 'slice_hrn') - - if len(slice_list) is 0: - raise SliverDoesNotExist("%s slice_hrn" % (slice_hrn)) - - #Used for fetching the user info witch comes along the slice info - one_slice = slice_list[0] - - - #Make a list of all the nodes hostnames in use for this slice - slice_nodes_list = [] - #for single_slice in slice_list: - #for node in single_slice['node_ids']: - #slice_nodes_list.append(node['hostname']) - for node in one_slice: - slice_nodes_list.append(node['hostname']) - - #Get all the corresponding nodes details - nodes_all = self.slab_api.GetNodes({'hostname':slice_nodes_list}, - ['node_id', 'hostname','site','boot_state']) - nodeall_byhostname = dict([(one_node['hostname'], one_node) \ - for one_node in nodes_all]) - - - - for single_slice in slice_list: - - #For compatibility - top_level_status = 'empty' - result = {} - result.fromkeys(\ - ['geni_urn','pl_login','geni_status','geni_resources'], None) - result['pl_login'] = one_slice['reg_researchers']['hrn'] - logger.debug("Slabdriver - sliver_status Sliver status \ - urn %s hrn %s single_slice %s \r\n " \ - %(slice_urn, slice_hrn, single_slice)) - - if 'node_ids' not in single_slice: - #No job in the slice - result['geni_status'] = top_level_status - result['geni_resources'] = [] - return result - - top_level_status = 'ready' - - #A job is running on Senslab for this slice - # report about the local nodes that are in the slice only - - result['geni_urn'] = slice_urn - - resources = [] - for node in single_slice['node_ids']: - res = {} - #res['slab_hostname'] = node['hostname'] - #res['slab_boot_state'] = node['boot_state'] - - res['pl_hostname'] = node['hostname'] - res['pl_boot_state'] = \ - nodeall_byhostname[node['hostname']]['boot_state'] - #res['pl_last_contact'] = strftime(self.time_format, \ - #gmtime(float(timestamp))) - sliver_id = Xrn(slice_urn, type='slice', \ - id=nodeall_byhostname[node['hostname']]['node_id'], \ - authority=self.hrn).urn - - res['geni_urn'] = sliver_id - node_name = node['hostname'] - if nodeall_byhostname[node_name]['boot_state'] == 'Alive': - - res['geni_status'] = 'ready' - else: - res['geni_status'] = 'failed' - top_level_status = 'failed' - - res['geni_error'] = '' - - resources.append(res) - - result['geni_status'] = top_level_status - result['geni_resources'] = resources - logger.debug("SLABDRIVER \tsliver_statusresources %s res %s "\ - %(resources,res)) - return result - - @staticmethod - def get_user_record(hrn): - """ - Returns the user record based on the hrn from the SFA DB . - - :param hrn: user's hrn - :type hrn: string - :return : user record from SFA database - :rtype: RegUser - - """ - return dbsession.query(RegRecord).filter_by(hrn = hrn).first() - - - def testbed_name (self): - """ - Returns testbed's name. - - :rtype: string - """ - return self.hrn - - # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory - def aggregate_version (self): - """ - - Returns the testbed's supported rspec advertisement and - request versions. - - :rtype: dict - """ - version_manager = VersionManager() - ad_rspec_versions = [] - request_rspec_versions = [] - for rspec_version in version_manager.versions: - if rspec_version.content_type in ['*', 'ad']: - ad_rspec_versions.append(rspec_version.to_dict()) - if rspec_version.content_type in ['*', 'request']: - request_rspec_versions.append(rspec_version.to_dict()) - return { - 'testbed':self.testbed_name(), - 'geni_request_rspec_versions': request_rspec_versions, - 'geni_ad_rspec_versions': ad_rspec_versions, - } - - - - def _get_requested_leases_list(self, rspec): - """ - Process leases in rspec depending on the rspec version (format) - type. Find the lease requests in the rspec and creates - a lease request list with the mandatory information ( nodes, - start time and duration) of the valid leases (duration above or equal - to the senslab experiment minimum duration). - - :param rspec: rspec request received. - :type rspec: RSpec - :return: list of lease requests found in the rspec - :rtype: list - """ - requested_lease_list = [] - for lease in rspec.version.get_leases(): - single_requested_lease = {} - logger.debug("SLABDRIVER.PY \tcreate_sliver lease %s " %(lease)) - - if not lease.get('lease_id'): - if get_authority(lease['component_id']) == \ - self.slab_api.root_auth: - single_requested_lease['hostname'] = \ - slab_xrn_to_hostname(\ - lease.get('component_id').strip()) - single_requested_lease['start_time'] = \ - lease.get('start_time') - single_requested_lease['duration'] = lease.get('duration') - #Check the experiment's duration is valid before adding - #the lease to the requested leases list - duration_in_seconds = \ - int(single_requested_lease['duration'])*60 - if duration_in_seconds > self.slab_api.GetMinExperimentDurationInSec() : - requested_lease_list.append(single_requested_lease) - - return requested_lease_list - - @staticmethod - def _group_leases_by_start_time(requested_lease_list): - """ - Create dict of leases by start_time, regrouping nodes reserved - at the same time, for the same amount of time so as to - define one job on OAR. - - :param requested_lease_list: list of leases - :type requested_lease_list: list - :return: Dictionary with key = start time, value = list of leases - with the same start time. - :rtype: dictionary - """ - - requested_job_dict = {} - for lease in requested_lease_list: - - #In case it is an asap experiment start_time is empty - if lease['start_time'] == '': - lease['start_time'] = '0' - - if lease['start_time'] not in requested_job_dict: - if isinstance(lease['hostname'], str): - lease['hostname'] = [lease['hostname']] - - - requested_job_dict[lease['start_time']] = lease - - else : - job_lease = requested_job_dict[lease['start_time']] - if lease['duration'] == job_lease['duration'] : - job_lease['hostname'].append(lease['hostname']) - - return requested_job_dict - - def _process_requested_jobs(self, rspec): - """ - Turns the requested leases and information into a dictionary - of requested jobs, grouped by starting time. - - :param rspec: RSpec received - :type rspec : RSpec - :rtype: dictionary - """ - requested_lease_list = self._get_requested_leases_list(rspec) - logger.debug("SLABDRIVER _process_requested_jobs requested_lease_list \ - %s"%(requested_lease_list)) - job_dict = self._group_leases_by_start_time(requested_lease_list) - logger.debug("SLABDRIVER _process_requested_jobs job_dict\ - %s"%(job_dict)) - - return job_dict - - def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, \ - users, options): - """ - Answer to CreateSliver. - Creates the leases and slivers for the users from the information - found in the rspec string. - Launch experiment on OAR if the requested leases is valid. Delete - no longer requested leases. - - - :param creds: user's credentials - :type creds: string - :param users: user record list - :type users: list - :param options: - :type options: - - :return: a valid Rspec for the slice which has just been - modified. - :rtype: RSpec - - - """ - aggregate = SlabAggregate(self) - - slices = SlabSlices(self) - peer = slices.get_peer(slice_hrn) - sfa_peer = slices.get_sfa_peer(slice_hrn) - slice_record = None - - if not isinstance(creds, list): - creds = [creds] - - if users: - slice_record = users[0].get('slice_record', {}) - logger.debug("SLABDRIVER.PY \t ===============create_sliver \t\ - creds %s \r\n \r\n users %s" \ - %(creds, users)) - slice_record['user'] = {'keys':users[0]['keys'], \ - 'email':users[0]['email'], \ - 'hrn':slice_record['reg-researchers'][0]} - # parse rspec - rspec = RSpec(rspec_string) - logger.debug("SLABDRIVER.PY \t create_sliver \trspec.version \ - %s slice_record %s users %s" \ - %(rspec.version,slice_record, users)) - - - # ensure site record exists? - # ensure slice record exists - #Removed options to verify_slice SA 14/08/12 - sfa_slice = slices.verify_slice(slice_hrn, slice_record, peer, \ - sfa_peer) - - # ensure person records exists - #verify_persons returns added persons but since the return value - #is not used - slices.verify_persons(slice_hrn, sfa_slice, users, peer, \ - sfa_peer, options=options) - #requested_attributes returned by rspec.version.get_slice_attributes() - #unused, removed SA 13/08/12 - #rspec.version.get_slice_attributes() - - logger.debug("SLABDRIVER.PY create_sliver slice %s " %(sfa_slice)) - - # add/remove slice from nodes - - #requested_slivers = [node.get('component_id') \ - #for node in rspec.version.get_nodes_with_slivers()\ - #if node.get('authority_id') is self.slab_api.root_auth] - #l = [ node for node in rspec.version.get_nodes_with_slivers() ] - #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \ - #requested_slivers %s listnodes %s" \ - #%(requested_slivers,l)) - #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12. - #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer) - - - requested_job_dict = self._process_requested_jobs(rspec) - - - logger.debug("SLABDRIVER.PY \tcreate_sliver requested_job_dict %s "\ - %(requested_job_dict)) - #verify_slice_leases returns the leases , but the return value is unused - #here. Removed SA 13/08/12 - slices.verify_slice_leases(sfa_slice, \ - requested_job_dict, peer) - - return aggregate.get_rspec(slice_xrn=slice_urn, \ - login=sfa_slice['login'], version=rspec.version) - - - def delete_sliver (self, slice_urn, slice_hrn, creds, options): - """ - Deletes the lease associated with the slice hrn and the credentials - if the slice belongs to senslab. Answer to DeleteSliver. - - :return: 1 if the slice to delete was not found on senslab, - True if the deletion was successful, False otherwise otherwise. - - .. note:: Should really be named delete_leases because senslab does - not have any slivers, but only deals with leases. However, SFA api only - have delete_sliver define so far. SA 13.05/2013 - """ - - sfa_slice_list = self.slab_api.GetSlices(slice_filter = slice_hrn, \ - slice_filter_type = 'slice_hrn') - - if not sfa_slice_list: - return 1 - - #Delete all leases in the slice - for sfa_slice in sfa_slice_list: - - - logger.debug("SLABDRIVER.PY delete_sliver slice %s" %(sfa_slice)) - slices = SlabSlices(self) - # determine if this is a peer slice - - peer = slices.get_peer(slice_hrn) - - logger.debug("SLABDRIVER.PY delete_sliver peer %s \ - \r\n \t sfa_slice %s " %(peer, sfa_slice)) - try: - - self.slab_api.DeleteSliceFromNodes(sfa_slice) - return True - except : - return False - - - def list_resources (self, slice_urn, slice_hrn, creds, options): - """ - List resources from the senslab aggregate and returns a Rspec - advertisement with resources found when slice_urn and slice_hrn are None - (in case of resource discovery). - If a slice hrn and urn are provided, list experiment's slice - nodes in a rspec format. Answer to ListResources. - Caching unused. - :param options: options used when listing resources (list_leases, info, - geni_available) - :return: rspec string in xml - :rtype: string - """ - - #cached_requested = options.get('cached', True) - - version_manager = VersionManager() - # get the rspec's return format from options - rspec_version = \ - version_manager.get_version(options.get('geni_rspec_version')) - version_string = "rspec_%s" % (rspec_version) - - #panos adding the info option to the caching key (can be improved) - if options.get('info'): - version_string = version_string + "_" + \ - options.get('info', 'default') - - # Adding the list_leases option to the caching key - if options.get('list_leases'): - version_string = version_string + "_" + \ - options.get('list_leases', 'default') - - # Adding geni_available to caching key - if options.get('geni_available'): - version_string = version_string + "_" + \ - str(options.get('geni_available')) - - # look in cache first - #if cached_requested and self.cache and not slice_hrn: - #rspec = self.cache.get(version_string) - #if rspec: - #logger.debug("SlabDriver.ListResources: \ - #returning cached advertisement") - #return rspec - - #panos: passing user-defined options - aggregate = SlabAggregate(self) - - rspec = aggregate.get_rspec(slice_xrn=slice_urn, \ - version=rspec_version, options=options) - - # cache the result - #if self.cache and not slice_hrn: - #logger.debug("Slab.ListResources: stores advertisement in cache") - #self.cache.add(version_string, rspec) - - return rspec - - - def list_slices (self, creds, options): - """ - Answer to ListSlices. - List slices belonging to senslab, returns slice urns list. - No caching used. Options unused but are defined in the SFA method - api prototype. - - :return: slice urns list - :rtype: list - - """ - # look in cache first - #if self.cache: - #slices = self.cache.get('slices') - #if slices: - #logger.debug("PlDriver.list_slices returns from cache") - #return slices - - # get data from db - - slices = self.slab_api.GetSlices() - logger.debug("SLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" %(slices)) - slice_hrns = [slab_slice['hrn'] for slab_slice in slices] - - slice_urns = [hrn_to_urn(slice_hrn, 'slice') \ - for slice_hrn in slice_hrns] - - # cache the result - #if self.cache: - #logger.debug ("SlabDriver.list_slices stores value in cache") - #self.cache.add('slices', slice_urns) - - return slice_urns - - - def register (self, sfa_record, hrn, pub_key): - """ - Adding new user, slice, node or site should not be handled - by SFA. - - ..warnings:: should not be used. Different components are in charge of - doing this task. Adding nodes = OAR - Adding users = LDAP Senslab - Adding slice = Import from LDAP users - Adding site = OAR - - :param sfa_record: record provided by the client of the - Register API call. - :type sfa_record: dict - """ - return -1 - - - def update (self, old_sfa_record, new_sfa_record, hrn, new_key): - """No site or node record update allowed in Senslab. - The only modifications authorized here are key deletion/addition - on an existing user and password change. - On an existing user, CAN NOT BE MODIFIED: - 'first_name', 'last_name', 'email' - DOES NOT EXIST IN SENSLAB: - 'phone', 'url', 'bio','title', 'accepted_aup', - A slice is bound to its user, so modifying the user's ssh key should - modify the slice's GID after an import procedure. - - :param old_sfa_record: what is in the db for this hrn - :param new_sfa_record: what was passed to the Update call - - ..seealso:: update in driver.py. - """ - - pointer = old_sfa_record['pointer'] - old_sfa_record_type = old_sfa_record['type'] - - # new_key implemented for users only - if new_key and old_sfa_record_type not in [ 'user' ]: - raise UnknownSfaType(old_sfa_record_type) - - - if old_sfa_record_type == "user": - update_fields = {} - all_fields = new_sfa_record - for key in all_fields.keys(): - if key in ['key', 'password']: - update_fields[key] = all_fields[key] - - - if new_key: - # must check this key against the previous one if it exists - persons = self.slab_api.GetPersons([old_sfa_record]) - person = persons[0] - keys = [person['pkey']] - #Get all the person's keys - keys_dict = self.slab_api.GetKeys(keys) - - # Delete all stale keys, meaning the user has only one key - #at a time - #TODO: do we really want to delete all the other keys? - #Is this a problem with the GID generation to have multiple - #keys? SA 30/05/13 - key_exists = False - if key in keys_dict: - key_exists = True - else: - #remove all the other keys - for key in keys_dict: - self.slab_api.DeleteKey(person, key) - self.slab_api.AddPersonKey(person, \ - {'sshPublicKey': person['pkey']},{'sshPublicKey': new_key} ) - #self.slab_api.AddPersonKey(person, {'key_type': 'ssh', \ - #'key': new_key}) - return True - - - def remove (self, sfa_record): - """ - Removes users only. Mark the user as disabled in - LDAP. The user and his slice are then deleted from the db by running an - import on the registry. - - - - :param sfa_record: record is the existing sfa record in the db - :type sfa_record: dict - - ..warning::As fas as the slice is concerned, here only the leases are - removed from the slice. The slice is record itself is not removed from - the db. - TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013, - - TODO: return boolean for the slice part - """ - sfa_record_type = sfa_record['type'] - hrn = sfa_record['hrn'] - if sfa_record_type == 'user': - - #get user from senslab ldap - person = self.slab_api.GetPersons(sfa_record) - #No registering at a given site in Senslab. - #Once registered to the LDAP, all senslab sites are - #accesible. - if person : - #Mark account as disabled in ldap - return self.slab_api.DeletePerson(sfa_record) - - elif sfa_record_type == 'slice': - if self.slab_api.GetSlices(slice_filter = hrn, \ - slice_filter_type = 'slice_hrn'): - ret = self.slab_api.DeleteSlice(sfa_record) - - - - return True - - diff --git a/sfa/senslab/slabpostgres.py b/sfa/senslab/slabpostgres.py deleted file mode 100644 index ca217760..00000000 --- a/sfa/senslab/slabpostgres.py +++ /dev/null @@ -1,177 +0,0 @@ -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - -from sfa.util.config import Config -from sfa.util.sfalogging import logger - -from sqlalchemy import Column, Integer, String -from sqlalchemy import Table, MetaData -from sqlalchemy.ext.declarative import declarative_base - -from sqlalchemy.dialects import postgresql - -from sqlalchemy.exc import NoSuchTableError - - -#Dict holding the columns names of the table as keys -#and their type, used for creation of the table -slice_table = {'record_id_user': 'integer PRIMARY KEY references X ON DELETE \ -CASCADE ON UPDATE CASCADE','oar_job_id':'integer DEFAULT -1', \ -'record_id_slice':'integer', 'slice_hrn':'text NOT NULL'} - -#Dict with all the specific senslab tables -tablenames_dict = {'slab_xp': slice_table} - - -SlabBase = declarative_base() - - - -class SenslabXP (SlabBase): - """ SQL alchemy class to manipulate slice_senslab table in - slab_sfa database. - - """ - __tablename__ = 'slab_xp' - - - slice_hrn = Column(String) - job_id = Column(Integer, primary_key = True) - end_time = Column(Integer, nullable = False) - - - #oar_job_id = Column( Integer,default = -1) - #node_list = Column(postgresql.ARRAY(String), nullable =True) - - def __init__ (self, slice_hrn =None, job_id=None, end_time=None): - """ - Defines a row of the slice_senslab table - """ - if slice_hrn: - self.slice_hrn = slice_hrn - if job_id : - self.job_id = job_id - if end_time: - self.end_time = end_time - - - def __repr__(self): - """Prints the SQLAlchemy record to the format defined - by the function. - """ - result = " +# Copyright (C) 2012 by +# # # Copyright: See COPYING file that comes with this distribution # ########################################################################### -#LDAP import -from sfa.senslab.LDAPapi import LDAPapi +#LDAP import +from sfa.iotlab.LDAPapi import LDAPapi import ldap.modlist as modlist - #logger sfa from sfa.util.sfalogging import logger #OAR imports from datetime import datetime -from sfa.senslab.OARrestapi import OARrestapi +from sfa.iotlab.OARrestapi import OARrestapi -#Test slabdriver -from sfa.senslab.slabdriver import SlabDriver +#Test iotlabdriver +from sfa.iotlab.iotlabdriver import IotlabDriver from sfa.util.config import Config @@ -26,9 +25,9 @@ import os import sys - + def parse_options(): - + #arguments supplied if len(sys.argv) > 1 : options_list = sys.argv[1:] @@ -38,12 +37,12 @@ def parse_options(): valid_options_dict = {} value_list = [] #Passing options to the script should be done like this : - #-10 OAR -2 SlabDriver + #-10 OAR -2 IotlabDriver for option in options_list: if option in supported_options: - #update the values used for the fonctions associated + #update the values used for the fonctions associated #with the options - + valid_options_dict[option] = value_list #empty the values list for next option value_list = [] @@ -54,16 +53,16 @@ def parse_options(): print "value_list", value_list - return valid_options_dict - + return valid_options_dict + def TestLdap(job_id = None): logger.setLevelDebug() ldap_server = LDAPapi() ret = ldap_server.conn.connect(bind=True) - ldap_server.conn.close() + ldap_server.conn.close() print "TEST ldap_server.conn.connect(bind=True)" , ret - + ret = ldap_server.conn.connect(bind=False) ldap_server.conn.close() print "TEST ldap_server.conn.connect(bind=False)", ret @@ -71,14 +70,14 @@ def TestLdap(job_id = None): ret = ldap_server.LdapSearch() print "TEST ldap_server.LdapSearch ALL", ret - + ret = ldap_server.LdapSearch('(uid=avakian)', []) print "\r\n TEST ldap_server.LdapSearch ids = avakian", ret password = ldap_server.generate_password() - print "\r\n TEST generate_password ", password - + print "\r\n TEST generate_password ", password + maxi = ldap_server.find_max_uidNumber() print "\r\n TEST find_max_uidNumber " , maxi @@ -87,315 +86,328 @@ def TestLdap(job_id = None): data['first_name'] = "Tim" data['givenName'] = data['first_name'] data['mail'] = "robin@arkham.fr" - + record = {} - record['hrn'] = 'senslab.drake' + record['hrn'] = 'iotlab.drake' record['last_name'] = "Drake" record['first_name'] = "Tim" record['mail'] = "robin@arkham.fr" - - + + login = ldap_server.generate_login(data) print "\r\n Robin \tgenerate_login ", ret, login - + ret = ldap_server.LdapAddUser(data) print "\r\n Robin \tLdapAddUser ", ret req_ldap = '(uid=' + login + ')' ret = ldap_server.LdapSearch(req_ldap, []) print "\r\n Robin \tldap_server.LdapSearch ids = %s %s" % (login, ret) - + password = "Thridrobin" enc = ldap_server.encrypt_password(password) print "\r\n Robin \tencrypt_password ", enc - + ret = ldap_server.LdapModifyUser(record, {'userPassword':enc}) print "\r\n Robin \tChange password LdapModifyUser ", ret - + #dn = 'uid=' + login + ',' + ldap_server.baseDN #ret = ldap_server.LdapDelete(dn) #print "\r\n Robin \tLdapDelete ", ret - + datanight = {} datanight['last_name'] = "Grayson" datanight['first_name'] = "Dick" datanight['givenName'] = datanight['first_name'] datanight['mail'] = "nightwing@arkham.fr" - - + + record_night = {} - record_night['hrn'] = 'senslab.grayson' + record_night['hrn'] = 'iotlab.grayson' record_night['last_name'] = datanight['last_name'] record_night['first_name'] = datanight['first_name'] record_night['mail'] = datanight['mail'] - + ret = ldap_server.LdapFindUser(record_night) print "\r\n Nightwing \tldap_server.LdapFindUser %s : %s" % (record_night, ret) - + #ret = ldap_server.LdapSearch('(uid=grayson)', []) #print "\r\n Nightwing \tldap_server.LdapSearch ids = %s %s" %('grayson',ret ) #ret = ldap_server.LdapAddUser(datanight) - #print "\r\n Nightwing \tLdapAddUser ", ret - + #print "\r\n Nightwing \tLdapAddUser ", ret + #ret = ldap_server.LdapResetPassword(record_night) #print "\r\n Nightwing \tLdapResetPassword de %s : %s" % (record_night, ret) - + ret = ldap_server.LdapDeleteUser(record_night) - print "\r\n Nightwing \tLdapDeleteUser ", ret - - + print "\r\n Nightwing \tLdapDeleteUser ", ret + + #record_avakian = {} - #record_avakian['hrn']= 'senslab.avakian' + #record_avakian['hrn']= 'iotlab.avakian' #record_avakian['last_name'] = 'avakian' #record_avakian['first_name'] = 'sandrine' #record_avakian['mail'] = 'sandrine.avakian@inria.fr' - #pubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAwSUkJ+cr3xM47h8lFkIXJoJhg4wHakTaLJmgTXkzvUmQsQeFB2MjUZ6WAelMXj/EFz2+XkK+bcWNXwfbrLptJQ+XwGpPZlu9YV/kzO63ghVrAyEg0+p7Pn1TO9f1ZYg4R6JfP/3qwH1AsE+X3PNpIewsuEIKwd2wUCJDf5RXJTpl39GizcBFemrRqgs0bdqAN/vUT9YvtWn8fCYR5EfJHVXOK8P1KmnbuGZpk7ryz21pDMlgw13+8aYB+LPkxdv5zG54A5c6o9N3zOCblvRFWaNBqathS8y04cOYWPmyu+Q0Xccwi7vM3Ktm8RoJw+raQNwsmneJOm6KXKnjoOQeiQ== savakian@sfa2.grenoble.senslab.info" + #pubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAwSUkJ+cr3xM47h8lFkIXJoJhg4wHakTaLJmgTXkzvUmQsQeFB2MjUZ6WAelMXj/EFz2+XkK+bcWNXwfbrLptJQ+XwGpPZlu9YV/kzO63ghVrAyEg0+p7Pn1TO9f1ZYg4R6JfP/3qwH1AsE+X3PNpIewsuEIKwd2wUCJDf5RXJTpl39GizcBFemrRqgs0bdqAN/vUT9YvtWn8fCYR5EfJHVXOK8P1KmnbuGZpk7ryz21pDMlgw13+8aYB+LPkxdv5zG54A5c6o9N3zOCblvRFWaNBqathS8y04cOYWPmyu+Q0Xccwi7vM3Ktm8RoJw+raQNwsmneJOm6KXKnjoOQeiQ== savakian@sfa2.grenoble.iotlab.info" #ret = ldap_server.LdapModifyUser(record_night, {'sshPublicKey':pubkey}) - #print "\r\n Sandrine \tChange pubkey LdapModifyUser ", ret - + #print "\r\n Sandrine \tChange pubkey LdapModifyUser ", ret + #record_myslice = {} - #record_myslice['hrn']= 'senslab.myslice' + #record_myslice['hrn']= 'iotlab.myslice' #record_myslice['last_name'] = 'myslice' #record_myslice['first_name'] = 'myslice' #record_myslice['mail'] = 'nturro@inria.fr' #pubkeymyslice = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAuyRPwn8PZxjdhu+ciRuPyM0eVBn7XS7i3tym9F30UVhaCd09a/UEmGn7WJZdfsxV3hXqG1Wc766FEst97NuzHzELSuvy/rT96J0UHG4wae4pnzOLd6NwFdZh7pkPsgHMHxK9ALVE68Puu+EDSOB5bBZ9Q624wCIGxEpmuS/+X+dDBTKgG5Hi0WA1uKJwhLSbbXb38auh4FlYgXPsdpljTIJatt+zGL0Zsy6fdrsVRc5W8kr3/SmE4OMNyabKBNyxioSEuYhRSjoQAHnYoevEjZniP8IzscKK7qwelzGUfnJEzexikhsQamhAFti2ReiFfoHBRZxnSc49ioH7Kaci5w== root@rhoecos3.ipv6.lip6.fr" - + #pubkeytestuser = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYS8tzufciTm6GdNUGHQc64OfTxFebMYUwh/Jl04IPTvjjr26uakbM0M2v33HxZ5Q7PnmPN9pB/w+a+f7a7J4cNs/tApOMg2hb6UrLaOrdnDMOs4KZlfElyDsF3Zx5QwxPYvzsKADAbDVoX4NF9PttuDLdm2l3nLSvm89jfla00GBg+K8grdOCHyYZVX/Wt7kxhXDK3AidQhKJgn+iD5GxvtWMBE+7S5kJGdRW1W10lSLBW3+VNsCrKJB2s8L55Xz/l2HNBScU7T0VcMQJrFxEXKzLPagZsMz0lfLzHESoGHIZ3Tz85DfECbTtMxLts/4KoAEc3EE+PYr2VDeAggDx testuser@myslice" - - + + #password = "ReptileFight" #enc = ldap_server.encrypt_password(password) #print "\r\n sandrine \tencrypt_password ", enc - + #ret = ldap_server.LdapModifyUser(record_avakian, {'userPassword':enc}) - #print "\r\n sandrine \tChange password LdapModifyUser ", ret + #print "\r\n sandrine \tChange password LdapModifyUser ", ret return def get_stuff(oar, uri): import httplib - import json + import json headers = {} - data = json.dumps({}) - - headers['X-REMOTE_IDENT'] = 'avakian' - + data = json.dumps({}) + + headers['X-REMOTE_IDENT'] = 'avakian' headers['content-length'] = '0' #seems that it does not work if we don't add this - + conn = httplib.HTTPConnection(oar.oarserver['ip'], oar.oarserver['port']) conn.request("GET", uri, data , headers ) - resp = ( conn.getresponse()).read() + resp = (conn.getresponse()).read() conn.close() - + js = json.loads(resp) return js - - + + def TestOAR(job_id = None): - print "JOB_ID", job_id + print "JOB_ID", job_id if isinstance(job_id, list) : if len(job_id) >= 1: job_id = job_id[0] else: job_id = '1' else: - job_id = '1' - print "JOB_ID", job_id + job_id = '1' + print "JOB_ID", job_id oar = OARrestapi() - jobs = oar.parser.SendRequest("GET_reserved_nodes", username = 'avakian') + jobs = oar.parser.SendRequest("GET_reserved_nodes", username = 'avakian') print "\r\n OAR GET_reserved_nodes ", jobs - - - jobs = oar.parser.SendRequest("GET_jobs") + + + jobs = oar.parser.SendRequest("GET_jobs") print "\r\n OAR GET_jobs ", jobs - - + + jobs = oar.parser.SendRequest("GET_jobs_id", job_id, 'avakian') print "\r\n OAR GET_jobs_id ", jobs - - uri = '/oarapi/jobs/details.json?state=Running,Waiting,Launching&user=avakian' + + uri = '/oarapi/jobs/details.json?state=Running,Waiting,Launching&user=avakian' raw_json = get_stuff(oar, uri) print "\r\nOAR ", uri, raw_json, "\r\n KKK \t", raw_json.keys() - + uri = '/oarapi/jobs/' + job_id +'.json' - raw_json = get_stuff(oar, uri) + raw_json = get_stuff(oar, uri) print "\r\n OAR ", uri, raw_json, "\r\n KKK \t", raw_json.keys() - + uri = '/oarapi/jobs/' + job_id + '/resources.json' raw_json = get_stuff(oar, uri) print "\r\n OAR ", uri, raw_json, "\r\n KKK \t", raw_json.keys() - + time_format = "%Y-%m-%d %H:%M:%S" - + server_timestamp, server_tz = oar.parser.SendRequest("GET_timezone") - + print "\r\n OAR GetTimezone ", server_timestamp, server_tz print(datetime.fromtimestamp(int(server_timestamp)).strftime('%Y-%m-%d %H:%M:%S')) uri = '/oarapi/resources/full.json' raw_json = get_stuff(oar, uri) print "\r\n OAR ", uri, raw_json, "\r\n KKK \t", raw_json.keys() - - uri = '/oarapi/jobs.json?user=avakian' + + uri = '/oarapi/jobs.json?user=avakian' raw_json = get_stuff(oar, uri) print "\r\nOAR ", uri, raw_json, "\r\n KKK \t", raw_json.keys() return - - - -def TestSlabDriver(job_id = None): + + +def TestImporter(arg=None): + iotlabdriver = IotlabDriver(Config()) + + nodes_listdict = iotlabdriver.iotlab_api.GetNodes() + sites_listdict = iotlabdriver.iotlab_api.GetSites() + nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict]) + + # from sfa.importer.iotlabimporter import IotlabImporter + # importer = IotlabImporter() + +def TestIotlabDriver(job_id = None): if job_id is None: job_id = 1 - + if isinstance(job_id, list) and len(job_id) == 1: job_id = job_id[0] - slabdriver = SlabDriver(Config()) - - #nodes = slabdriver.slab_api.GetReservedNodes() + iotlabdriver = IotlabDriver(Config()) + + #nodes = iotlabdriver.iotlab_api.GetReservedNodes() #print " \r\n \r\n GetReservedNodes", nodes - - #sl = slabdriver.slab_api.GetSlices(slice_filter='senslab.avakian_slice', slice_filter_type='slice_hrn') - #print "\r\n \r\nGetSlices", sl[0] - - #sl = slabdriver.slab_api.GetSlices(slice_filter='20', slice_filter_type='record_id_user') + + sl = iotlabdriver.iotlab_api.GetSlices(slice_filter='iotlab.avakian_slice', slice_filter_type='slice_hrn') + print "\r\n \r\nGetSlices", sl[0] + + #sl = iotlabdriver.iotlab_api.GetSlices(slice_filter='20', slice_filter_type='record_id_user') #print "\r\n \r\nGetSlices", sl - - #sl = slabdriver.slab_api.GetSlices() + + #sl = iotlabdriver.iotlab_api.GetSlices() #print "\r\n \r\nGetSlices", sl - - persons = slabdriver.slab_api.GetPersons() + + persons = iotlabdriver.iotlab_api.GetPersons() print "\r\n \r\n GetPersons", persons - - leases = slabdriver.slab_api.GetLeases(login='avakian') + + leases = iotlabdriver.iotlab_api.GetLeases(login='avakian') + print "\r\n \r\n GetLeases", leases + + leases = iotlabdriver.iotlab_api.GetLeases(lease_filter_dict={'slice_hrn':'iotlab.avakian_slice'}) print "\r\n \r\n GetLeases", leases - + leases = iotlabdriver.iotlab_api.GetLeases(lease_filter_dict={'t_from':1405070000}) + print "\r\n \r\n GetLeases", leases def TestSfi(filename = None): if filename is None: filename = "/home/savakian/flab-sfa/test_rspec/my_lyon_nodes.rspec" print " ================= SFI.PY RESOURCES =============", \ - os.system("sfi.py list senslab") - + os.system("sfi.py list iotlab") + print os.system("sfi.py resources") - print os.system("sfi.py resources -r slab") + print os.system("sfi.py resources -r iotlab") print os.system("sfi.py resources -l all") - - - print "================ SFI.PY RESOURCES -R SLAB -L ALL ============\r\n", \ - os.system("sfi.py resources -r slab -l all") - + + + print "================ SFI.PY RESOURCES -R IOTLAB -L ALL ============\r\n", \ + os.system("sfi.py resources -r iotlab -l all") + print "================ WRITING sfi.py resources -l all ===========\r\n", \ filename - + filename = filename.split(".")[0]+"_out.rspec" rspecfile = open(filename,"w") - r = os.popen("sfi.py resources -l all") + r = os.popen("sfi.py resources -l all") for i in r.readlines(): rspecfile.write(i) rspecfile.close() - + print " ================= SFI.PY SHOW SLICE ============= \r\n", \ - os.system("sfi.py resources senslab.avakian_slice") - + os.system("sfi.py resources iotlab.avakian_slice") + print " ================= SFI.PY SHOW USER =============\r\n", \ - os.system("sfi.py show senslab.avakian_slice") + os.system("sfi.py show iotlab.avakian_slice") print " ================= SFI.PY SHOW NODE =============\r\n", \ - os.system("sfi.py show senslab.avakian") + os.system("sfi.py show iotlab.avakian") print " ================= SFI.PY SLICES =============\r\n", \ - os.system("sfi.py show senslab.node6.devlille.senslab.info") + os.system("sfi.py show iotlab.node6.devlille.iotlab.info") print " ================= SFI.PY LIST SLICE =============\r\n", \ os.system("sfi.py slices") print " ================= SFI.PY STATUS SLICE =============\r\n", \ - os.system("sfi.py status senslab.avakian_slice") - + os.system("sfi.py status iotlab.avakian_slice") + print " ================= SFI.PY DELETE SLICE =============\r\n", \ - os.system("sfi.py delete senslab.avakian_slice") - + os.system("sfi.py delete iotlab.avakian_slice") + print " ================= SFI.PY CREATE SLICE =============\r\n", \ - os.system("sfi.py create senslab.avakian_slice \ + os.system("sfi.py create iotlab.avakian_slice \ /home/savakian/flab-sfa/test_rspec/my_lyon_nodes.rspec") - + def TestSQL(arg = None): from sfa.storage.model import make_record, RegSlice, RegRecord from sfa.storage.alchemy import dbsession - from sqlalchemy.orm.collections import InstrumentedList - - from sqlalchemy.orm import joinedload - - #solo_query_slice_list = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn='senslab.avakian_slice').first() + from sqlalchemy.orm.collections import InstrumentedList + + from sqlalchemy.orm import joinedload + + #solo_query_slice_list = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn='iotlab.avakian_slice').first() #print "\r\n \r\n =========== query_slice_list RegSlice \ - #joinedload('reg_researchers') senslab.avakian first \r\n \t ", \ + #joinedload('reg_researchers') iotlab.avakian first \r\n \t ", \ #solo_query_slice_list.__dict__ - - #query_slice_list = dbsession.query(RegSlice).options(joinedload('reg_researchers')).all() + + #query_slice_list = dbsession.query(RegSlice).options(joinedload('reg_researchers')).all() #print "\r\n \r\n =========== query_slice_list RegSlice \ #joinedload('reg_researchers') ALL \r\n \t", \ - #query_slice_list[0].__dict__ - + #query_slice_list[0].__dict__ + #return_slicerec_dictlist = [] #record = query_slice_list[0] - #print "\r\n \r\n =========== \r\n \t", record - + #print "\r\n \r\n =========== \r\n \t", record + #tmp = record.__dict__ - #print "\r\n \r\n =========== \r\n \t", tmp + #print "\r\n \r\n =========== \r\n \t", tmp #tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__ - #print "\r\n \r\n =========== \r\n \t", tmp + #print "\r\n \r\n =========== \r\n \t", tmp ##del tmp['reg_researchers']['_sa_instance_state'] #return_slicerec_dictlist.append(tmp) - + #print "\r\n \r\n =========== \r\n \t", return_slicerec_dictlist - + all_records = dbsession.query(RegRecord).all() - - #create hash by (type,hrn) - #used to know if a given record is already known to SFA - + + #create hash by (type,hrn) + #used to know if a given record is already known to SFA + records_by_type_hrn = \ dict ( [ ( (record.type,record.hrn) , record ) for record in all_records ] ) for (rec_type, rec) in records_by_type_hrn : if rec_type == 'user': - print >>sys.stderr,"\r\n SLABIMPORT \t keys %s rec %s \r\n" %(rec_type, rec ) - + print >>sys.stderr,"\r\n IOTLABIMPORT \t keys %s rec %s \r\n" %(rec_type, rec ) + users_rec_by_email = \ dict ( [ (record.email, record) for record in all_records if record.type == 'user' ] ) - - + + def RunAll( arg ): TestLdap() TestOAR() - TestSlabDriver() + TestIotlabDriver() TestSfi() - - + + supported_options = { 'OAR' : TestOAR, 'LDAP': TestLdap, - 'driver': TestSlabDriver, + 'driver': TestIotlabDriver, 'sfi':TestSfi, 'sql':TestSQL, - 'all' : RunAll } - + 'all' : RunAll, + 'import': TestImporter } + def main(): opts = parse_options() print opts for opt in opts: supported_options[opt](opts[opt]) - + if __name__ == "__main__": - main() \ No newline at end of file + main() \ No newline at end of file diff --git a/testbeds/iotlab/tests/sfi_client_tests.py b/testbeds/iotlab/tests/sfi_client_tests.py new file mode 100755 index 00000000..634707de --- /dev/null +++ b/testbeds/iotlab/tests/sfi_client_tests.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python +import sys +import os +from sfa.iotlab.LDAPapi import LDAPapi +from difflib import SequenceMatcher + +def parse_options(): + + #arguments supplied + if len(sys.argv) > 1 : + options_list = sys.argv[1:] + print options_list + rspec_rep = options_list[0] + return rspec_rep + else: + print "Must supply Rspecs directory ", sys.argv[1:] + return + + +rspec_dir = parse_options() +print "DIRECTORY SUPPLIED" , rspec_dir +rspec_filename_list = ['firexp_avakian_slice_iotlab.rspec', +'firexp_iotlab_slice_iotlab.rspec', +'iotlab_avakian_slice_iotlab2.rspec', +'iotlab_avakian_slice_plab.rspec', +'firexp_iotlab_slice_all.rspec', +'iotlab_avakian_slice_all.rspec', +'iotlab_avakian_slice_iotlab.rspec', +'iotlab_user_slice_iotlab.rspec', +'test_delete_all_leases.rspec'] + +rspec_filename_dict = { + ('iotlab_avakian', 'iotlab'): + "sfi.py create iotlab.avakian_slice " + rspec_dir + \ + 'iotlab_avakian_slice_iotlab.rspec', + + ('iotlab_avakian', 'iotlab2'): + "sfi.py create iotlab.avakian_slice " + rspec_dir + \ + 'iotlab_avakian_slice_iotlab2.rspec', + + ('firexp_user','iotlab'): + "sfi.py create firexp.flab.iotlab_slice " + rspec_dir + \ + 'firexp_iotlab_slice_iotlab.rspec', + + ('firexp_user', 'all'): + "sfi.py create firexp.flab.iotlab_slice "+ rspec_dir + \ + 'firexp_iotlab_slice_all.rspec', + + ('iotlab_user', 'iotlab'): + "sfi.py create iotlab.user_slice "+ rspec_dir + \ + 'iotlab_user_slice_iotlab.rspec', + + ('firexp_avakian','iotlab'): + "sfi.py create firexp.flab.avakian_slice " + rspec_dir + \ + 'firexp_avakian_slice_iotlab.rspec', + + ('iotlab_avakian', 'plab') : + "sfi.py create iotlab.avakian_slice " + rspec_dir + \ + 'iotlab_avakian_slice_plab.rspec', + + ('iotlab_avakian', 'all') : + "sfi.py create iotlab.avakian_slice " + rspec_dir + \ + 'iotlab_avakian_slice_all.rspec' + + } +# check if the firexp user (uid user) is already in LDAP +# in this is the case, delete it : +ldap_server = LDAPapi() +dn = 'uid=' + 'user' + ',' + ldap_server.baseDN +result = ldap_server.LdapSearch('(uid=user)', []) + +if result != []: + retval = ldap_server.LdapDelete(dn) + print "deleting firexp user : ", retval + + +print "config sfi" +with open ("/root/.sfi/sfi_config", "r") as sfi_config: + sfi_config_txt = [line for line in sfi_config] + +with open("/root/.sfi/sfi_config_iotlab", "r") as sfi_config_iotlab: + sfi_config_iotlab_txt = [line for line in sfi_config_iotlab] + +with open("/root/.sfi/sfi_config_firexp", "r") as sfi_config_firexp: + sfi_config_firexp_txt = [line for line in sfi_config_firexp] +# check that we are using the iotlab sfi configuration +result1 = SequenceMatcher(None, sfi_config_txt, sfi_config_iotlab_txt) + +result2 = SequenceMatcher(None, sfi_config_txt, sfi_config_firexp_txt) + +if result1.ratio() != 1.0: + os.system('cp /root/.sfi/sfi_config_iotlab /root/.sfi/sfi_config') + +os.system('cat /root/.sfi/sfi_config') +os.system('rm /root/tests_rspecs/iotlab_devlille_OUTPUT.rspec') + +print " ================= SFI.PY LIST IOTLAB =============" +os.system('sfi.py list iotlab') + + +print " ================= SFI.PY RESOURCES =============" +raw_input("Press Enter to continue...") +os.system('sfi.py resources') + + +print " ================= SFI.PY RESOURCES -R IOTLAB =============" +raw_input("Press Enter to continue...") +os.system('sfi.py resources -r iotlab') + + +print " ================= SFI.PY RESOURCES -L ALL =============" +raw_input("Press Enter to continue...") +os.system('sfi.py resources -l all') + +print " ================= SFI.PY RESOURCES -R IOTLAB -L ALL =============" +raw_input("Press Enter to continue...") +os.system('sfi.py resources -r iotlab -l all') + +print " ================= SFI.PY RESOURCES -O output rspec ===========" +os.system('sfi.py resources -o /root/tests_rspecs/iotlab_devlille_OUTPUT.rspec') + +print " ================= SFI.PY RESOURCES -L LEASES =============" +raw_input("Press Enter to continue...") +os.system('sfi.py resources -l leases') + + +print " ================= SFI.PY SHOW USER =============" +raw_input("Press Enter to continue...") +os.system('sfi.py show iotlab.avakian') + +print " ================= SFI.PY SHOW NODE =============" +os.system('sfi.py show m3-3.devgrenoble.iot-lab.info') + +print " ================= SFI.PY SLICES =============" +raw_input("Press Enter to continue...") +os.system('sfi.py slices') + +print " ================= SFI.PY STATUS SLICE =============" +os.system('sfi.py status iotlab.avakian_slice') + +print " ================= SFI.PY CREATE SLICE on iotlab only =============" +raw_input("Press Enter to continue...") +os.system( rspec_filename_dict[('iotlab_avakian','iotlab')]) + + +print " ================= SFI.PY RESOURCES -l all iotlab.avakian_slice =============" +raw_input("Press Enter to continue...") +os.system('sfi.py resources -l all iotlab.avakian_slice') + + +print " ================= SFI.PY DELETE SLICE =============" +raw_input("Press Enter to continue...") +os.system('sfi.py delete iotlab.avakian_slice') + + +print " ================= SFI.PY CREATE SLICE on iotlab and firexp =============" +raw_input("Press Enter to continue...") +os.system(rspec_filename_dict[('iotlab_avakian','all')]) + + +print " ================= SFI.PY RESOURCES -l all -r iotlab iotlab.avakian_slice =============" +raw_input("Press Enter to continue...") +os.system('sfi.py resources -l all -r iotlab iotlab.avakian_slice') + + +print " =================SFI.PY RESOURCES -L LEASES -R IOTLAB ============== " +os.system('sfi.py resources -r iotlab -l leases') + + +print " ================= SFI.PY DELETE SLICE =============" +raw_input("Press Enter to continue...") +os.system('sfi.py delete iotlab.avakian_slice') + +print "\r\n \r\n" + +print " *********changing to firexp sfi config ***************" +os.system('cp /root/.sfi/sfi_config_firexp /root/.sfi/sfi_config') + + + +print " ================= SFI.PY CREATE SLICE on iotlab and firexp =============" +raw_input("Press Enter to continue...") +os.system(rspec_filename_dict[('firexp_user','all')]) + + +print " ================= SFI.PY SHOW SLICE =============" +raw_input("Press Enter to continue...") +os.system('sfi.py show firexp.flab.iotlab_slice') + + +print " ================= SFI.PY RESOURCES -l leases firexp.flab.iotlab_slice =============" +raw_input("Press Enter to continue...") +os.system('sfi.py resources -l leases firexp.flab.iotlab_slice') + + +print " ================= SFI.PY RESOURCES firexp.flab.iotlab_slice =============" +raw_input("Press Enter to continue...") +os.system('sfi.py resources firexp.flab.iotlab_slice') + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/README b/testbeds/iotlab/tests/tests_rspecs/README new file mode 100644 index 00000000..bf10e22b --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/README @@ -0,0 +1,36 @@ +Rspec file names +====================== +Rspec file names are constructed as follows : + slice name used in this rspec + network in which the reserved nodes are + +Networks can be : iotlab, plab, all (iotlab + plab) + +Slices and users +================= +user: +login iotlab : user +hrn iotlab: iotlab.user +hrn firexp: firexp.flab.iotlab_user +slice iotlab: iotlab.user_slice +slice firexp : firexp.flab.iotlab_slice + + +This special test user comes from Firexp and is considered as an +external user coming from a federated testbedd for Iotlab. + +user: +login iotlab: avakian +slice iotlab: iotlab.avakian_slice +hrn firexp : firexp.flab.avakian (?) +slice firexp : firexp.flab.avakian_slice (?) + +This user comes from iotlab. + + +Leases +====== + +The starting time of the leases in those RSpec files are +usually set to be in 2014, so that we don't have to keep the +date in mind and check that we are not scheduling a lease +in the past. diff --git a/testbeds/iotlab/tests/tests_rspecs/firexp_avakian_slice_iotlab.rspec b/testbeds/iotlab/tests/tests_rspecs/firexp_avakian_slice_iotlab.rspec new file mode 100644 index 00000000..121a4dd7 --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/firexp_avakian_slice_iotlab.rspec @@ -0,0 +1,23 @@ + + + + + wsn430-12.devlille.iot-lab.info + + TRUE + + + + + a8-11.devgrenoble.iot-lab.info + + TRUE + + + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/firexp_iotlab_slice_all.rspec b/testbeds/iotlab/tests/tests_rspecs/firexp_iotlab_slice_all.rspec new file mode 100644 index 00000000..c2e25589 --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/firexp_iotlab_slice_all.rspec @@ -0,0 +1,38 @@ + + + + + effet.pl.sophia.inria.fr + TRUE + + + planetlab.test.plab.effet + + + + + + + + + wsn430-12.devlille.iot-lab.info + + TRUE + + + + + a8-11.devgrenoble.iot-lab.info + + TRUE + + + + + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/firexp_iotlab_slice_iotlab.rspec b/testbeds/iotlab/tests/tests_rspecs/firexp_iotlab_slice_iotlab.rspec new file mode 100644 index 00000000..8bb3aee6 --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/firexp_iotlab_slice_iotlab.rspec @@ -0,0 +1,23 @@ + + + + + wsn430-12.devlille.iot-lab.info + + TRUE + + + + + a8-11.devgrenoble.iot-lab.info + + TRUE + + + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_all.rspec b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_all.rspec new file mode 100644 index 00000000..2feb7bc0 --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_all.rspec @@ -0,0 +1,38 @@ + + + + + effet.pl.sophia.inria.fr + TRUE + + + planetlab.test.plab.effet + + + + + + + + + wsn430-12.devlille.iot-lab.info + + TRUE + + + + + a8-11.devgrenoble.iot-lab.info + + TRUE + + + + + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab.rspec b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab.rspec new file mode 100644 index 00000000..dd582056 --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab.rspec @@ -0,0 +1,23 @@ + + + + + wsn430-12.devlille.iot-lab.info + + TRUE + + + + + a8-11.devgrenoble.iot-lab.info + + TRUE + + + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab2.rspec b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab2.rspec new file mode 100644 index 00000000..c22b9e87 --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_iotlab2.rspec @@ -0,0 +1,35 @@ + + + + + wsn430-8.devlille.iot-lab.info + + TRUE + + + + + wsn430-5.devlille.iot-lab.info + + TRUE + + + + + + + effet.pl.sophia.inria.fr + FALSE + + x86_64 + f14 + onelab + planetlab.test.plab.effet + + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_plab.rspec b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_plab.rspec new file mode 100644 index 00000000..6bdccf77 --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/iotlab_avakian_slice_plab.rspec @@ -0,0 +1,16 @@ + + + + + effet.pl.sophia.inria.fr + TRUE + + + planetlab.test.plab.effet + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/iotlab_user_slice_iotlab.rspec b/testbeds/iotlab/tests/tests_rspecs/iotlab_user_slice_iotlab.rspec new file mode 100644 index 00000000..6a283f3d --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/iotlab_user_slice_iotlab.rspec @@ -0,0 +1,23 @@ + + + + + wsn430-12.devlille.iot-lab.info + + TRUE + + + + + "a8-11.devgrenoble.iot-lab.info + + TRUE + + + + + + + + + diff --git a/testbeds/iotlab/tests/tests_rspecs/test_delete_all_leases.rspec b/testbeds/iotlab/tests/tests_rspecs/test_delete_all_leases.rspec new file mode 100644 index 00000000..ad1d659c --- /dev/null +++ b/testbeds/iotlab/tests/tests_rspecs/test_delete_all_leases.rspec @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/testbeds/senslab/tests/bash_testsuite b/testbeds/senslab/tests/bash_testsuite deleted file mode 100755 index 6bcfd504..00000000 --- a/testbeds/senslab/tests/bash_testsuite +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -if (( ! $# == 2 )) -then - echo " Usage : bash_test takes 2 arguments : one jobid and one of the following:" - echo " LDAP/ OAR / driver " - echo $# - exit -fi - -sfi.py list senslab2 -echo " ================= SFI.PY RESOURCES =============" -sfi.py resources - -echo " ================= SFI.PY RESOURCES -R SLAB =============" -sfi.py resources -r slab - -echo " ================= SFI.PY RESOURCES -L ALL =============" -sfi.py resources -l all - -echo " ================= SFI.PY RESOURCES -R SLAB -L ALL =============" -sfi.py resources -r slab -l all - -echo " ================= SFI.PY RESOURCES -L ALL > avakian_adv.rspec =============" -sfi.py resources -l all > /home/savakian/flab-sfa/avakian_adv.rspec - -echo " ================= SFI.PY RESOURCES avakian_adv.rspec =============" -sfi.py resources senslab2.avakian_slice - - -echo " ================= SFI.PY SHOW SLICE =============" -sfi.py show senslab2.avakian_slice - -echo " ================= SFI.PY SHOW USER =============" -sfi.py show senslab2.avakian - -echo " ================= SFI.PY SHOW NODE =============" -sfi.py show senslab2.node67.grenoble.senslab.info - -echo " ================= SFI.PY SLICES =============" -sfi.py slices - -echo " ================= SFI.PY STATUS SLICE =============" -sfi.py status senslab2.avakian_slice - -echo " ================= SFI.PY CREATE SLICE =============" -sfi.py create senslab2.avakian_slice /home/savakian/flab-sfa/avakian_adv.rspec - -# echo " ================= SFI.PY DELETE SLICE =============" -# sfi.py delete senslab2.avakian_slice - -echo "\r\n" -echo " PYTHON TEST ", $1, $2