From: Sandrine Avakian Date: Thu, 21 Jun 2012 14:44:52 +0000 (+0200) Subject: Merge branch 'master' into senslab2 X-Git-Tag: sfa-2.1-24~3^2~152^2 X-Git-Url: http://git.onelab.eu/?p=sfa.git;a=commitdiff_plain;h=cedf37c8661ec0b05349274962bf2db9cfa6a24d;hp=-c Merge branch 'master' into senslab2 Conflicts: setup.py --- cedf37c8661ec0b05349274962bf2db9cfa6a24d diff --combined setup.py index eafced43,a0c222a0..96cb8cb1 --- a/setup.py +++ b/setup.py @@@ -9,7 -9,7 +9,7 @@@ from glob import glo import shutil from distutils.core import setup - scripts = glob("sfa/clientbin/*.py") + \ + scripts = glob("clientbin/*.py") + \ [ 'config/sfa-config-tty', 'config/gen-sfa-cm-config.py', @@@ -22,26 -22,22 +22,31 @@@ packages = [ 'sfa', - 'sfa/openstack', 'sfa/trust', 'sfa/storage', 'sfa/util', - 'sfa/client', 'sfa/server', 'sfa/methods', 'sfa/generic', 'sfa/managers', 'sfa/importer', + + ++ + 'sfa/senslab', + - 'sfa/planetlab', ++ ++ ++ + 'sfa/rspecs', 'sfa/rspecs/elements', 'sfa/rspecs/elements/versions', 'sfa/rspecs/versions', + 'sfa/client', + 'sfa/planetlab', + 'sfa/openstack', + 'sfa/federica', 'sfatables', 'sfatables/commands', 'sfatables/processors', diff --combined sfa/client/sfi.py index 1b0f9b75,c8082dc9..43e5b985 --- a/sfa/client/sfi.py +++ b/sfa/client/sfi.py @@@ -1,7 -1,6 +1,6 @@@ # # sfi.py - basic SFA command-line client - # the actual binary in sfa/clientbin essentially runs main() - # this module is used in sfascan + # this module is also used in sfascan # import sys @@@ -9,6 -8,7 +8,7 @@@ sys.path.append('.' import os, os.path import socket + import re import datetime import codecs import pickle @@@ -23,8 -23,9 +23,9 @@@ from sfa.trust.gid import GI from sfa.trust.credential import Credential from sfa.trust.sfaticket import SfaTicket + from sfa.util.faults import SfaInvalidArgument from sfa.util.sfalogging import sfi_logger - from sfa.util.xrn import get_leaf, get_authority, hrn_to_urn + from sfa.util.xrn import get_leaf, get_authority, hrn_to_urn, Xrn from sfa.util.config import Config from sfa.util.version import version_core from sfa.util.cache import Cache @@@ -43,6 -44,30 +44,30 @@@ from sfa.client.return_value import Ret CM_PORT=12346 # utility methods here + def optparse_listvalue_callback(option, option_string, value, parser): + setattr(parser.values, option.dest, value.split(',')) + + # a code fragment that could be helpful for argparse which unfortunately is + # available with 2.7 only, so this feels like too strong a requirement for the client side + #class ExtraArgAction (argparse.Action): + # def __call__ (self, parser, namespace, values, option_string=None): + # would need a try/except of course + # (k,v)=values.split('=') + # d=getattr(namespace,self.dest) + # d[k]=v + ##### + #parser.add_argument ("-X","--extra",dest='extras', default={}, action=ExtraArgAction, + # help="set extra flags, testbed dependent, e.g. --extra enabled=true") + + def optparse_dictvalue_callback (option, option_string, value, parser): + try: + (k,v)=value.split('=',1) + d=getattr(parser.values, option.dest) + d[k]=v + except: + parser.print_help() + sys.exit(1) + # display methods def display_rspec(rspec, format='rspec'): if format in ['dns']: @@@ -72,7 -97,7 +97,7 @@@ def display_records(recordList, dump=Fa def display_record(record, dump=False): if dump: - record.dump() + record.dump(sort=True) else: info = record.getdict() print "%s (%s)" % (info['hrn'], info['type']) @@@ -154,8 -179,44 +179,44 @@@ def save_record_to_file(filename, recor f.close() return + # minimally check a key argument + def check_ssh_key (key): + good_ssh_key = r'^.*(?:ssh-dss|ssh-rsa)[ ]+[A-Za-z0-9+/=]+(?: .*)?$' + return re.match(good_ssh_key, key, re.IGNORECASE) # load methods + def load_record_from_opts(options): + record_dict = {} + if hasattr(options, 'xrn') and options.xrn: + if hasattr(options, 'type') and options.type: + xrn = Xrn(options.xrn, options.type) + else: + xrn = Xrn(options.xrn) + record_dict['urn'] = xrn.get_urn() + record_dict['hrn'] = xrn.get_hrn() + record_dict['type'] = xrn.get_type() + if hasattr(options, 'key') and options.key: + try: + pubkey = open(options.key, 'r').read() + except IOError: + pubkey = options.key + if not check_ssh_key (pubkey): + raise SfaInvalidArgument(name='key',msg="Could not find file, or wrong key format") + record_dict['keys'] = [pubkey] + if hasattr(options, 'slices') and options.slices: + record_dict['slices'] = options.slices + if hasattr(options, 'researchers') and options.researchers: + record_dict['researcher'] = options.researchers + if hasattr(options, 'email') and options.email: + record_dict['email'] = options.email + if hasattr(options, 'pis') and options.pis: + record_dict['pi'] = options.pis + + # handle extra settings + record_dict.update(options.extras) + + return Record(dict=record_dict) + def load_record_from_file(filename): f=codecs.open(filename, encoding="utf-8", mode="r") xml_string = f.read() @@@ -257,6 -318,30 +318,30 @@@ class Sfi parser = OptionParser(usage="sfi [sfi_options] %s [cmd_options] %s" \ % (command, self.available_dict[command])) + if command in ("add", "update"): + parser.add_option('-x', '--xrn', dest='xrn', metavar='', help='object hrn/urn (mandatory)') + parser.add_option('-t', '--type', dest='type', metavar='', help='object type', default=None) + parser.add_option('-e', '--email', dest='email', default="", help="email (mandatory for users)") + # use --extra instead + # parser.add_option('-u', '--url', dest='url', metavar='', default=None, help="URL, useful for slices") + # parser.add_option('-d', '--description', dest='description', metavar='', + # help='Description, useful for slices', default=None) + parser.add_option('-k', '--key', dest='key', metavar='', help='public key string or file', + default=None) + parser.add_option('-s', '--slices', dest='slices', metavar='', help='slice xrns', + default='', type="str", action='callback', callback=optparse_listvalue_callback) + parser.add_option('-r', '--researchers', dest='researchers', metavar='', + help='slice researchers', default='', type="str", action='callback', + callback=optparse_listvalue_callback) + parser.add_option('-p', '--pis', dest='pis', metavar='', help='Principal Investigators/Project Managers', + default='', type="str", action='callback', callback=optparse_listvalue_callback) + # use --extra instead + # parser.add_option('-f', '--firstname', dest='firstname', metavar='', help='user first name') + # parser.add_option('-l', '--lastname', dest='lastname', metavar='', help='user last name') + parser.add_option ('-X','--extra',dest='extras',default={},type='str',metavar="", + action="callback", callback=optparse_dictvalue_callback, nargs=1, + help="set extra/testbed-dependent flags, e.g. --extra enabled=true") + # user specifies remote aggregate/sm/component if command in ("resources", "slices", "create", "delete", "start", "stop", "restart", "shutdown", "get_ticket", "renew", "status"): @@@ -284,8 -369,12 +369,12 @@@ help="display format ([xml]|dns|ip)", default="xml", choices=("xml", "dns", "ip")) #panos: a new option to define the type of information about resources a user is interested in - parser.add_option("-i", "--info", dest="info", + parser.add_option("-i", "--info", dest="info", help="optional component information", default=None) + # a new option to retreive or not reservation-oriented RSpecs (leases) + parser.add_option("-l", "--list_leases", dest="list_leases", type="choice", + help="Retreive or not reservation-oriented RSpecs ([resources]|leases|all )", + choices=("all", "resources", "leases"), default="resources") # 'create' does return the new rspec, makes sense to save that too @@@ -737,7 -826,7 +826,7 @@@ or version information about sfi itsel self.logger.error("No record of type %s"% options.type) records = [ Record(dict=record_dict) for record_dict in record_dicts ] for record in records: - if (options.format == "text"): record.dump() + if (options.format == "text"): record.dump(sort=True) else: print record.save_as_xml() if options.file: save_records_to_file(options.file, record_dicts, options.fileformat) @@@ -746,29 -835,50 +835,50 @@@ def add(self, options, args): "add record into registry from xml file (Register)" auth_cred = self.my_authority_credential_string() - if len(args)!=1: + record_dict = {} + if len(args) > 0: + record_filepath = args[0] + rec_file = self.get_record_file(record_filepath) + record_dict.update(load_record_from_file(rec_file).todict()) + if options: + record_dict.update(load_record_from_opts(options).todict()) + # we should have a type by now + if 'type' not in record_dict : self.print_help() sys.exit(1) - record_filepath = args[0] - rec_file = self.get_record_file(record_filepath) - record = load_record_from_file(rec_file).todict() - return self.registry().Register(record, auth_cred) + # this is still planetlab dependent.. as plc will whine without that + # also, it's only for adding + if record_dict['type'] == 'user': + if not 'first_name' in record_dict: + record_dict['first_name'] = record_dict['hrn'] + if 'last_name' not in record_dict: + record_dict['last_name'] = record_dict['hrn'] + return self.registry().Register(record_dict, auth_cred) def update(self, options, args): "update record into registry from xml file (Update)" - if len(args)!=1: + record_dict = {} + if len(args) > 0: + record_filepath = args[0] + rec_file = self.get_record_file(record_filepath) + record_dict.update(load_record_from_file(rec_file).todict()) + if options: + record_dict.update(load_record_from_opts(options).todict()) + # at the very least we need 'type' here + if 'type' not in record_dict: self.print_help() sys.exit(1) - rec_file = self.get_record_file(args[0]) - record = load_record_from_file(rec_file) - if record.type == "user": - if record.hrn == self.user: + + # don't translate into an object, as this would possibly distort + # user-provided data; e.g. add an 'email' field to Users + if record_dict['type'] == "user": + if record_dict['hrn'] == self.user: cred = self.my_credential_string else: cred = self.my_authority_credential_string() - elif record.type in ["slice"]: + elif record_dict['type'] in ["slice"]: try: - cred = self.slice_credential_string(record.hrn) + cred = self.slice_credential_string(record_dict['hrn']) except ServerException, e: # XXX smbaker -- once we have better error return codes, update this # to do something better than a string compare @@@ -776,13 -886,12 +886,12 @@@ cred = self.my_authority_credential_string() else: raise - elif record.type in ["authority"]: + elif record_dict['type'] in ["authority"]: cred = self.my_authority_credential_string() - elif record.type == 'node': + elif record_dict['type'] == 'node': cred = self.my_authority_credential_string() else: - raise "unknown record type" + record.type - record_dict = record.todict() + raise "unknown record type" + record_dict['type'] return self.registry().Update(record_dict, cred) def remove(self, options, args): @@@ -849,6 -958,8 +958,8 @@@ or with an slice hrn, shows currently p api_options['geni_slice_urn'] = hrn_to_urn(hrn, 'slice') if options.info: api_options['info'] = options.info + if options.list_leases: + api_options['list_leases'] = options.list_leases if options.current: if options.current == True: api_options['cached'] = False @@@ -923,7 -1034,6 +1034,7 @@@ rspec.filter({'component_manager_id': server_version['urn']}) rspec = RSpecConverter.to_pg_rspec(rspec.toxml(), content_type='request') else: + print >>sys.stderr, "\r\n \r\n \r\n WOOOOOO" users = sfa_users_arg(user_records, slice_record) # do not append users, keys, or slice tags. Anything diff --combined sfa/planetlab/plslices.py index 90da4046,54cfaa19..49f2fbb3 --- a/sfa/planetlab/plslices.py +++ b/sfa/planetlab/plslices.py @@@ -1,6 -1,5 +1,6 @@@ from types import StringTypes from collections import defaultdict +import sys from sfa.util.sfatime import utcparse, datetime_to_epoch from sfa.util.sfalogging import logger @@@ -131,11 -130,13 +131,11 @@@ class PlSlices # slice belongs to out local plc or a myplc peer. We will assume it # is a local site, unless we find out otherwise peer = None - # get this slice's authority (site) slice_authority = get_authority(hrn) # get this site's authority (sfa root authority or sub authority) site_authority = get_authority(slice_authority).lower() - # check if we are already peered with this site_authority, if so peers = self.driver.shell.GetPeers({}, ['peer_id', 'peername', 'shortname', 'hrn_root']) for peer_record in peers: @@@ -158,6 -159,25 +158,25 @@@ return sfa_peer + def verify_slice_leases(self, slice, requested_leases, kept_leases, peer): + + leases = self.driver.shell.GetLeases({'name':slice['name']}, ['lease_id']) + current_leases = [lease['lease_id'] for lease in leases] + deleted_leases = list(set(current_leases).difference(kept_leases)) + + try: + if peer: + self.driver.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname']) + deleted=self.driver.shell.DeleteLeases(deleted_leases) + for lease in requested_leases: + added=self.driver.shell.AddLeases(lease['hostname'], slice['name'], int(lease['t_from']), int(lease['t_until'])) + + except: + logger.log_exc('Failed to add/remove slice leases') + + return leases + + def verify_slice_nodes(self, slice, requested_slivers, peer): nodes = self.driver.shell.GetNodes(slice['node_ids'], ['node_id', 'hostname', 'interface_ids']) @@@ -354,6 -374,7 +373,7 @@@ users_by_site = defaultdict(list) users_dict = {} for user in users: + user['urn'] = user['urn'].lower() hrn, type = urn_to_hrn(user['urn']) username = get_leaf(hrn) login_base = PlXrn(xrn=user['urn']).pl_login_base() @@@ -361,6 -382,7 +381,7 @@@ user['site'] = login_base if 'email' in user: + user['email'] = user['email'].lower() users_by_email[user['email']] = user users_dict[user['email']] = user else: diff --combined sfa/util/xrn.py index 60b57b98,083e620f..19b1fc18 --- a/sfa/util/xrn.py +++ b/sfa/util/xrn.py @@@ -22,7 -22,7 +22,7 @@@ #---------------------------------------------------------------------- import re - +import sys from sfa.util.faults import SfaAPIError # for convenience and smoother translation - we should get rid of these functions eventually @@@ -92,16 -92,21 +92,21 @@@ class Xrn return True return False + ########## basic tools on URNs URN_PREFIX = "urn:publicid:IDN" + URN_PREFIX_lower = "urn:publicid:idn" + + @staticmethod + def is_urn (text): + return text.lower().startswith(Xrn.URN_PREFIX_lower) - ########## basic tools on URNs @staticmethod def urn_full (urn): - if urn.startswith(Xrn.URN_PREFIX): return urn + if Xrn.is_urn(urn): return urn else: return Xrn.URN_PREFIX+urn @staticmethod def urn_meaningful (urn): - if urn.startswith(Xrn.URN_PREFIX): return urn[len(Xrn.URN_PREFIX):] + if Xrn.is_urn(urn): return urn[len(Xrn.URN_PREFIX):] else: return urn @staticmethod def urn_split (urn): @@@ -116,9 -121,8 +121,9 @@@ # provide either urn, or (hrn + type) def __init__ (self, xrn, type=None): if not xrn: xrn = "" + # user has specified xrn : guess if urn or hrn - if xrn.startswith(Xrn.URN_PREFIX): + if Xrn.is_urn(xrn): self.hrn=None self.urn=xrn self.urn_to_hrn() @@@ -150,8 -154,7 +155,8 @@@ # self.authority keeps a list if not hasattr(self,'authority'): self.authority=Xrn.hrn_auth_list(self.hrn) - + + def get_leaf(self): self._normalize() return self.leaf @@@ -182,7 -185,7 +187,7 @@@ """ # if not self.urn or not self.urn.startswith(Xrn.URN_PREFIX): - if not self.urn.startswith(Xrn.URN_PREFIX): + if not Xrn.is_urn(self.urn): raise SfaAPIError, "Xrn.urn_to_hrn" parts = Xrn.urn_split(self.urn) @@@ -216,7 -219,7 +221,7 @@@ """ # if not self.hrn or self.hrn.startswith(Xrn.URN_PREFIX): - if self.hrn.startswith(Xrn.URN_PREFIX): + if Xrn.is_urn(self.hrn): raise SfaAPIError, "Xrn.hrn_to_urn, hrn=%s"%self.hrn if self.type and self.type.startswith('authority'):