Merge branch 'master' into senslab2
authorSandrine Avakian <sandrine.avakian@inria.fr>
Thu, 21 Jun 2012 14:44:52 +0000 (16:44 +0200)
committerSandrine Avakian <sandrine.avakian@inria.fr>
Thu, 21 Jun 2012 14:44:52 +0000 (16:44 +0200)
Conflicts:
setup.py

1  2 
setup.py
sfa/client/sfi.py
sfa/planetlab/plslices.py
sfa/util/xrn.py

diff --combined setup.py
+++ b/setup.py
@@@ -9,7 -9,7 +9,7 @@@ from glob import glo
  import shutil
  from distutils.core import setup
  
- scripts = glob("sfa/clientbin/*.py") + \
+ scripts = glob("clientbin/*.py") + \
      [ 
      'config/sfa-config-tty',
      'config/gen-sfa-cm-config.py',
  
  packages = [
      'sfa', 
-     'sfa/openstack',
      'sfa/trust',
      'sfa/storage',
      'sfa/util', 
-     'sfa/client',
      'sfa/server',
      'sfa/methods',
      'sfa/generic',
      'sfa/managers',
      'sfa/importer',
-     'sfa/planetlab',
 +
 +
++
 +    'sfa/senslab',
 +
++
++
++
 +
      'sfa/rspecs',
      'sfa/rspecs/elements',
      'sfa/rspecs/elements/versions',
      'sfa/rspecs/versions',
+     'sfa/client',
+     'sfa/planetlab',
+     'sfa/openstack',
+     'sfa/federica',
      'sfatables',
      'sfatables/commands',
      'sfatables/processors',
diff --combined sfa/client/sfi.py
@@@ -1,7 -1,6 +1,6 @@@
  #
  # sfi.py - basic SFA command-line client
- # the actual binary in sfa/clientbin essentially runs main()
- # this module is used in sfascan
+ # this module is also used in sfascan
  #
  
  import sys
@@@ -9,6 -8,7 +8,7 @@@ sys.path.append('.'
  
  import os, os.path
  import socket
+ import re
  import datetime
  import codecs
  import pickle
@@@ -23,8 -23,9 +23,9 @@@ from sfa.trust.gid import GI
  from sfa.trust.credential import Credential
  from sfa.trust.sfaticket import SfaTicket
  
+ from sfa.util.faults import SfaInvalidArgument
  from sfa.util.sfalogging import sfi_logger
- from sfa.util.xrn import get_leaf, get_authority, hrn_to_urn
+ from sfa.util.xrn import get_leaf, get_authority, hrn_to_urn, Xrn
  from sfa.util.config import Config
  from sfa.util.version import version_core
  from sfa.util.cache import Cache
@@@ -43,6 -44,30 +44,30 @@@ from sfa.client.return_value import Ret
  CM_PORT=12346
  
  # utility methods here
+ def optparse_listvalue_callback(option, option_string, value, parser):
+     setattr(parser.values, option.dest, value.split(','))
+ # a code fragment that could be helpful for argparse which unfortunately is 
+ # available with 2.7 only, so this feels like too strong a requirement for the client side
+ #class ExtraArgAction  (argparse.Action):
+ #    def __call__ (self, parser, namespace, values, option_string=None):
+ # would need a try/except of course
+ #        (k,v)=values.split('=')
+ #        d=getattr(namespace,self.dest)
+ #        d[k]=v
+ #####
+ #parser.add_argument ("-X","--extra",dest='extras', default={}, action=ExtraArgAction,
+ #                     help="set extra flags, testbed dependent, e.g. --extra enabled=true")
+     
+ def optparse_dictvalue_callback (option, option_string, value, parser):
+     try:
+         (k,v)=value.split('=',1)
+         d=getattr(parser.values, option.dest)
+         d[k]=v
+     except:
+         parser.print_help()
+         sys.exit(1)
  # display methods
  def display_rspec(rspec, format='rspec'):
      if format in ['dns']:
@@@ -72,7 -97,7 +97,7 @@@ def display_records(recordList, dump=Fa
  
  def display_record(record, dump=False):
      if dump:
-         record.dump()
+         record.dump(sort=True)
      else:
          info = record.getdict()
          print "%s (%s)" % (info['hrn'], info['type'])
@@@ -154,8 -179,44 +179,44 @@@ def save_record_to_file(filename, recor
      f.close()
      return
  
+ # minimally check a key argument
+ def check_ssh_key (key):
+     good_ssh_key = r'^.*(?:ssh-dss|ssh-rsa)[ ]+[A-Za-z0-9+/=]+(?: .*)?$'
+     return re.match(good_ssh_key, key, re.IGNORECASE)
  
  # load methods
+ def load_record_from_opts(options):
+     record_dict = {}
+     if hasattr(options, 'xrn') and options.xrn:
+         if hasattr(options, 'type') and options.type:
+             xrn = Xrn(options.xrn, options.type)
+         else:
+             xrn = Xrn(options.xrn)
+         record_dict['urn'] = xrn.get_urn()
+         record_dict['hrn'] = xrn.get_hrn()
+         record_dict['type'] = xrn.get_type()
+     if hasattr(options, 'key') and options.key:
+         try:
+             pubkey = open(options.key, 'r').read()
+         except IOError:
+             pubkey = options.key
+         if not check_ssh_key (pubkey):
+             raise SfaInvalidArgument(name='key',msg="Could not find file, or wrong key format")
+         record_dict['keys'] = [pubkey]
+     if hasattr(options, 'slices') and options.slices:
+         record_dict['slices'] = options.slices
+     if hasattr(options, 'researchers') and options.researchers:
+         record_dict['researcher'] = options.researchers
+     if hasattr(options, 'email') and options.email:
+         record_dict['email'] = options.email
+     if hasattr(options, 'pis') and options.pis:
+         record_dict['pi'] = options.pis
+     # handle extra settings
+     record_dict.update(options.extras)
+     
+     return Record(dict=record_dict)
  def load_record_from_file(filename):
      f=codecs.open(filename, encoding="utf-8", mode="r")
      xml_string = f.read()
@@@ -257,6 -318,30 +318,30 @@@ class Sfi
          parser = OptionParser(usage="sfi [sfi_options] %s [cmd_options] %s" \
                                       % (command, self.available_dict[command]))
  
+         if command in ("add", "update"):
+             parser.add_option('-x', '--xrn', dest='xrn', metavar='<xrn>', help='object hrn/urn (mandatory)')
+             parser.add_option('-t', '--type', dest='type', metavar='<type>', help='object type', default=None)
+             parser.add_option('-e', '--email', dest='email', default="",  help="email (mandatory for users)") 
+ # use --extra instead
+ #            parser.add_option('-u', '--url', dest='url', metavar='<url>', default=None, help="URL, useful for slices") 
+ #            parser.add_option('-d', '--description', dest='description', metavar='<description>', 
+ #                              help='Description, useful for slices', default=None)
+             parser.add_option('-k', '--key', dest='key', metavar='<key>', help='public key string or file', 
+                               default=None)
+             parser.add_option('-s', '--slices', dest='slices', metavar='<slices>', help='slice xrns',
+                               default='', type="str", action='callback', callback=optparse_listvalue_callback)
+             parser.add_option('-r', '--researchers', dest='researchers', metavar='<researchers>', 
+                               help='slice researchers', default='', type="str", action='callback', 
+                               callback=optparse_listvalue_callback)
+             parser.add_option('-p', '--pis', dest='pis', metavar='<PIs>', help='Principal Investigators/Project Managers',
+                               default='', type="str", action='callback', callback=optparse_listvalue_callback)
+ # use --extra instead
+ #            parser.add_option('-f', '--firstname', dest='firstname', metavar='<firstname>', help='user first name')
+ #            parser.add_option('-l', '--lastname', dest='lastname', metavar='<lastname>', help='user last name')
+             parser.add_option ('-X','--extra',dest='extras',default={},type='str',metavar="<EXTRA_ASSIGNS>",
+                                action="callback", callback=optparse_dictvalue_callback, nargs=1,
+                                help="set extra/testbed-dependent flags, e.g. --extra enabled=true")
          # user specifies remote aggregate/sm/component                          
          if command in ("resources", "slices", "create", "delete", "start", "stop", 
                         "restart", "shutdown",  "get_ticket", "renew", "status"):
                               help="display format ([xml]|dns|ip)", default="xml",
                               choices=("xml", "dns", "ip"))
              #panos: a new option to define the type of information about resources a user is interested in
-           parser.add_option("-i", "--info", dest="info",
+             parser.add_option("-i", "--info", dest="info",
                                  help="optional component information", default=None)
+             # a new option to retreive or not reservation-oriented RSpecs (leases)
+             parser.add_option("-l", "--list_leases", dest="list_leases", type="choice",
+                                 help="Retreive or not reservation-oriented RSpecs ([resources]|leases|all )",
+                                 choices=("all", "resources", "leases"), default="resources")
  
  
          # 'create' does return the new rspec, makes sense to save that too
@@@ -737,7 -826,7 +826,7 @@@ or version information about sfi itsel
              self.logger.error("No record of type %s"% options.type)
          records = [ Record(dict=record_dict) for record_dict in record_dicts ]
          for record in records:
-             if (options.format == "text"):      record.dump()  
+             if (options.format == "text"):      record.dump(sort=True)  
              else:                               print record.save_as_xml() 
          if options.file:
              save_records_to_file(options.file, record_dicts, options.fileformat)
      def add(self, options, args):
          "add record into registry from xml file (Register)"
          auth_cred = self.my_authority_credential_string()
-         if len(args)!=1:
+         record_dict = {}
+         if len(args) > 0:
+             record_filepath = args[0]
+             rec_file = self.get_record_file(record_filepath)
+             record_dict.update(load_record_from_file(rec_file).todict())
+         if options:
+             record_dict.update(load_record_from_opts(options).todict())
+         # we should have a type by now
+         if 'type' not in record_dict :
              self.print_help()
              sys.exit(1)
-         record_filepath = args[0]
-         rec_file = self.get_record_file(record_filepath)
-         record = load_record_from_file(rec_file).todict()
-         return self.registry().Register(record, auth_cred)
+         # this is still planetlab dependent.. as plc will whine without that
+         # also, it's only for adding
+         if record_dict['type'] == 'user':
+             if not 'first_name' in record_dict:
+                 record_dict['first_name'] = record_dict['hrn']
+             if 'last_name' not in record_dict:
+                 record_dict['last_name'] = record_dict['hrn'] 
+         return self.registry().Register(record_dict, auth_cred)
      
      def update(self, options, args):
          "update record into registry from xml file (Update)"
-         if len(args)!=1:
+         record_dict = {}
+         if len(args) > 0:
+             record_filepath = args[0]
+             rec_file = self.get_record_file(record_filepath)
+             record_dict.update(load_record_from_file(rec_file).todict())
+         if options:
+             record_dict.update(load_record_from_opts(options).todict())
+         # at the very least we need 'type' here
+         if 'type' not in record_dict:
              self.print_help()
              sys.exit(1)
-         rec_file = self.get_record_file(args[0])
-         record = load_record_from_file(rec_file)
-         if record.type == "user":
-             if record.hrn == self.user:
+         # don't translate into an object, as this would possibly distort
+         # user-provided data; e.g. add an 'email' field to Users
+         if record_dict['type'] == "user":
+             if record_dict['hrn'] == self.user:
                  cred = self.my_credential_string
              else:
                  cred = self.my_authority_credential_string()
-         elif record.type in ["slice"]:
+         elif record_dict['type'] in ["slice"]:
              try:
-                 cred = self.slice_credential_string(record.hrn)
+                 cred = self.slice_credential_string(record_dict['hrn'])
              except ServerException, e:
                 # XXX smbaker -- once we have better error return codes, update this
                 # to do something better than a string compare
                     cred = self.my_authority_credential_string()
                 else:
                     raise
-         elif record.type in ["authority"]:
+         elif record_dict['type'] in ["authority"]:
              cred = self.my_authority_credential_string()
-         elif record.type == 'node':
+         elif record_dict['type'] == 'node':
              cred = self.my_authority_credential_string()
          else:
-             raise "unknown record type" + record.type
-         record_dict = record.todict()
+             raise "unknown record type" + record_dict['type']
          return self.registry().Update(record_dict, cred)
    
      def remove(self, options, args):
@@@ -849,6 -958,8 +958,8 @@@ or with an slice hrn, shows currently p
              api_options['geni_slice_urn'] = hrn_to_urn(hrn, 'slice')
          if options.info:
              api_options['info'] = options.info
+         if options.list_leases:
+             api_options['list_leases'] = options.list_leases
          if options.current:
              if options.current == True:
                  api_options['cached'] = False
                  rspec.filter({'component_manager_id': server_version['urn']})
                  rspec = RSpecConverter.to_pg_rspec(rspec.toxml(), content_type='request')
              else:
 +                print >>sys.stderr, "\r\n \r\n \r\n WOOOOOO"
                  users = sfa_users_arg(user_records, slice_record)
  
          # do not append users, keys, or slice tags. Anything
@@@ -1,6 -1,5 +1,6 @@@
  from types import StringTypes
  from collections import defaultdict
 +import sys
  
  from sfa.util.sfatime import utcparse, datetime_to_epoch
  from sfa.util.sfalogging import logger
@@@ -131,11 -130,13 +131,11 @@@ class PlSlices
          # slice belongs to out local plc or a myplc peer. We will assume it 
          # is a local site, unless we find out otherwise  
          peer = None
 -
          # get this slice's authority (site)
          slice_authority = get_authority(hrn)
  
          # get this site's authority (sfa root authority or sub authority)
          site_authority = get_authority(slice_authority).lower()
 -
          # check if we are already peered with this site_authority, if so
          peers = self.driver.shell.GetPeers({}, ['peer_id', 'peername', 'shortname', 'hrn_root'])
          for peer_record in peers:
  
          return sfa_peer
  
+     def verify_slice_leases(self, slice, requested_leases, kept_leases, peer):
+         
+         leases = self.driver.shell.GetLeases({'name':slice['name']}, ['lease_id'])
+         current_leases = [lease['lease_id'] for lease in leases]
+         deleted_leases = list(set(current_leases).difference(kept_leases))
+         try:
+             if peer:
+                 self.driver.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname'])
+             deleted=self.driver.shell.DeleteLeases(deleted_leases)
+             for lease in requested_leases:
+                 added=self.driver.shell.AddLeases(lease['hostname'], slice['name'], int(lease['t_from']), int(lease['t_until']))
+         except: 
+             logger.log_exc('Failed to add/remove slice leases')
+         return leases
      def verify_slice_nodes(self, slice, requested_slivers, peer):
          
          nodes = self.driver.shell.GetNodes(slice['node_ids'], ['node_id', 'hostname', 'interface_ids'])
          users_by_site = defaultdict(list)
          users_dict = {} 
          for user in users:
+             user['urn'] = user['urn'].lower()
              hrn, type = urn_to_hrn(user['urn'])
              username = get_leaf(hrn)
              login_base = PlXrn(xrn=user['urn']).pl_login_base()
              user['site'] = login_base
  
              if 'email' in user:
+                 user['email'] = user['email'].lower() 
                  users_by_email[user['email']] = user
                  users_dict[user['email']] = user
              else:
diff --combined sfa/util/xrn.py
@@@ -22,7 -22,7 +22,7 @@@
  #----------------------------------------------------------------------
  
  import re
 -
 +import sys
  from sfa.util.faults import SfaAPIError
  
  # for convenience and smoother translation - we should get rid of these functions eventually 
@@@ -92,16 -92,21 +92,21 @@@ class Xrn
                  return True
          return False
  
+     ########## basic tools on URNs
      URN_PREFIX = "urn:publicid:IDN"
+     URN_PREFIX_lower = "urn:publicid:idn"
+     @staticmethod
+     def is_urn (text):
+         return text.lower().startswith(Xrn.URN_PREFIX_lower)
  
-     ########## basic tools on URNs
      @staticmethod
      def urn_full (urn):
-         if urn.startswith(Xrn.URN_PREFIX): return urn
+         if Xrn.is_urn(urn): return urn
          else: return Xrn.URN_PREFIX+urn
      @staticmethod
      def urn_meaningful (urn):
-         if urn.startswith(Xrn.URN_PREFIX): return urn[len(Xrn.URN_PREFIX):]
+         if Xrn.is_urn(urn): return urn[len(Xrn.URN_PREFIX):]
          else: return urn
      @staticmethod
      def urn_split (urn):
      # provide either urn, or (hrn + type)
      def __init__ (self, xrn, type=None):
          if not xrn: xrn = ""
 +       
          # user has specified xrn : guess if urn or hrn
-         if xrn.startswith(Xrn.URN_PREFIX):
+         if Xrn.is_urn(xrn):
              self.hrn=None
              self.urn=xrn
              self.urn_to_hrn()
          # self.authority keeps a list
          if not hasattr(self,'authority'): 
              self.authority=Xrn.hrn_auth_list(self.hrn)
 -
 +       
 +       
      def get_leaf(self):
          self._normalize()
          return self.leaf
          """
          
  #        if not self.urn or not self.urn.startswith(Xrn.URN_PREFIX):
-         if not self.urn.startswith(Xrn.URN_PREFIX):
+         if not Xrn.is_urn(self.urn):
              raise SfaAPIError, "Xrn.urn_to_hrn"
  
          parts = Xrn.urn_split(self.urn)
          """
  
  #        if not self.hrn or self.hrn.startswith(Xrn.URN_PREFIX):
-         if self.hrn.startswith(Xrn.URN_PREFIX):
+         if Xrn.is_urn(self.hrn):
              raise SfaAPIError, "Xrn.hrn_to_urn, hrn=%s"%self.hrn
  
          if self.type and self.type.startswith('authority'):