Merge branch 'geni-v3' into dbsession
authorThierry Parmentelat <thierry.parmentelat@inria.fr>
Thu, 14 Nov 2013 07:31:26 +0000 (08:31 +0100)
committerThierry Parmentelat <thierry.parmentelat@inria.fr>
Thu, 14 Nov 2013 07:31:26 +0000 (08:31 +0100)
dbsession at this point is almost in sync with geni-v3, let's integrate tehe rest of the deltas

1  2 
sfa/importer/plimporter.py
sfa/planetlab/plaggregate.py
sfa/planetlab/pldriver.py
sfa/planetlab/plslices.py

@@@ -24,9 -24,7 +24,9 @@@ from sfa.util.xrn import Xrn, get_leaf
  from sfa.trust.gid import create_uuid    
  from sfa.trust.certificate import convert_public_key, Keypair
  
 -from sfa.storage.alchemy import dbsession
 +# using global alchemy.session() here is fine 
 +# as importer is on standalone one-shot process
 +from sfa.storage.alchemy import global_dbsession
  from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, RegUser, RegKey
  
  from sfa.planetlab.plshell import PlShell    
@@@ -117,8 -115,8 +117,8 @@@ class PlImporter
                                             pointer=site['site_id'],
                                             authority=get_authority(site_hrn))
                  auth_record.just_created()
 -                dbsession.add(auth_record)
 -                dbsession.commit()
 +                global_dbsession.add(auth_record)
 +                global_dbsession.commit()
                  self.logger.info("PlImporter: Imported authority (vini site) %s"%auth_record)
                  self.remember_record ( site_record )
  
          shell = PlShell (config)
  
          ######## retrieve all existing SFA objects
 -        all_records = dbsession.query(RegRecord).all()
 +        all_records = global_dbsession.query(RegRecord).all()
  
          # create hash by (type,hrn) 
          # we essentially use this to know if a given record is already known to SFA 
  
          # start importing 
          for site in sites:
-             if site['name'].startswith('sfa.'):
+             if site['name'].startswith('sfa:'):
                  continue
  
              site_hrn = _get_site_hrn(interface_hrn, site)
                                                 pointer=site['site_id'],
                                                 authority=get_authority(site_hrn))
                      site_record.just_created()
 -                    dbsession.add(site_record)
 -                    dbsession.commit()
 +                    global_dbsession.add(site_record)
 +                    global_dbsession.commit()
                      self.logger.info("PlImporter: imported authority (site) : %s" % site_record) 
                      self.remember_record (site_record)
                  except:
                                                 pointer =node['node_id'],
                                                 authority=get_authority(node_hrn))
                          node_record.just_created()
 -                        dbsession.add(node_record)
 -                        dbsession.commit()
 +                        global_dbsession.add(node_record)
 +                        global_dbsession.commit()
                          self.logger.info("PlImporter: imported node: %s" % node_record)  
                          self.remember_record (node_record)
                      except:
                          else:
                              self.logger.warning("No key found for user %s"%user_record)
                          user_record.just_created()
 -                        dbsession.add (user_record)
 -                        dbsession.commit()
 +                        global_dbsession.add (user_record)
 +                        global_dbsession.commit()
                          self.logger.info("PlImporter: imported person: %s" % user_record)
                          self.remember_record ( user_record )
                      else:
                              user_record.just_updated()
                              self.logger.info("PlImporter: updated person: %s" % user_record)
                      user_record.email = person['email']
 -                    dbsession.commit()
 +                    global_dbsession.commit()
                      user_record.stale=False
                      # accumulate PIs - PLCAPI has a limitation that when someone has PI role
                      # this is valid for all sites she is in..
              # could be performed twice with the same person...
              # so hopefully we do not need to eliminate duplicates explicitly here anymore
              site_record.reg_pis = list(set(site_pis))
 -            dbsession.commit()
 +            global_dbsession.commit()
  
              # import slices
              for slice_id in site['slice_ids']:
                                                   pointer=slice['slice_id'],
                                                   authority=get_authority(slice_hrn))
                          slice_record.just_created()
 -                        dbsession.add(slice_record)
 -                        dbsession.commit()
 +                        global_dbsession.add(slice_record)
 +                        global_dbsession.commit()
                          self.logger.info("PlImporter: imported slice: %s" % slice_record)  
                          self.remember_record ( slice_record )
                      except:
                  # record current users affiliated with the slice
                  slice_record.reg_researchers = \
                      [ self.locate_by_type_pointer ('user',user_id) for user_id in slice['person_ids'] ]
 -                dbsession.commit()
 +                global_dbsession.commit()
                  slice_record.stale=False
  
          ### remove stale records
                  self.logger.warning("stale not found with %s"%record)
              if stale:
                  self.logger.info("PlImporter: deleting stale record: %s" % record)
 -                dbsession.delete(record)
 -                dbsession.commit()
 +                global_dbsession.delete(record)
 +                global_dbsession.commit()
@@@ -18,9 -18,10 +18,9 @@@ from sfa.rspecs.elements.lease import L
  from sfa.rspecs.elements.granularity import Granularity
  from sfa.rspecs.version_manager import VersionManager
  
- from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn, xrn_to_ext_slicename, top_auth
+ from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn, top_auth, hash_loginbase
  from sfa.planetlab.vlink import get_tc_rate
  from sfa.planetlab.topology import Topology
 -from sfa.storage.alchemy import dbsession
  from sfa.storage.model import SliverAllocation
  
  
@@@ -130,10 -131,14 +130,14 @@@ class PlAggregate
              else:  
                  slice_hrn = xrn.get_hrn()
                  top_auth_hrn = top_auth(slice_hrn)
+                 site_hrn = '.'.join(slice_hrn.split('.')[:-1])
+                 slice_part = slice_hrn.split('.')[-1]
                  if top_auth_hrn == self.driver.hrn:
-                     slice_name = hrn_to_pl_slicename(slice_hrn)
+                     login_base = slice_hrn.split('.')[-2][:12]
                  else:
-                     slice_name = xrn_to_ext_slicename(slice_hrn) 
+                     login_base = hash_loginbase(site_hrn)
+                 slice_name = '_'.join([login_base, slice_part])
                  names.add(slice_name)
  
          filter = {}
@@@ -10,6 -10,7 +10,6 @@@ from sfa.util.xrn import Xrn, hrn_to_ur
  from sfa.util.cache import Cache
  
  # one would think the driver should not need to mess with the SFA db, but..
 -from sfa.storage.alchemy import dbsession
  from sfa.storage.model import RegRecord, SliverAllocation
  from sfa.trust.credential import Credential
  
@@@ -24,7 -25,7 +24,7 @@@ from sfa.planetlab.plshell import PlShe
  import sfa.planetlab.peers as peers
  from sfa.planetlab.plaggregate import PlAggregate
  from sfa.planetlab.plslices import PlSlices
- from sfa.planetlab.plxrn import PlXrn, slicename_to_hrn, hostname_to_hrn, hrn_to_pl_slicename, xrn_to_hostname, xrn_to_ext_slicename, top_auth
+ from sfa.planetlab.plxrn import PlXrn, slicename_to_hrn, hostname_to_hrn, hrn_to_pl_slicename, xrn_to_hostname, top_auth, hash_loginbase
  
  
  def list_to_dict(recs, key):
@@@ -44,9 -45,8 +44,9 @@@ class PlDriver (Driver)
      # the cache instance is a class member so it survives across incoming requests
      cache = None
  
 -    def __init__ (self, config):
 -        Driver.__init__ (self, config)
 +    def __init__ (self, api):
 +        Driver.__init__ (self, api)
 +        config=api.config
          self.shell = PlShell (config)
          self.cache=None
          if config.SFA_AGGREGATE_CACHING:
                  if 'max_slices' not in pl_record:
                      pl_record['max_slices']=2
                  pointer = self.shell.AddSite(pl_record)
+                 self.shell.SetSiteHrn(int(pointer), hrn)
              else:
                  pointer = sites[0]['site_id']
  
              slices = self.shell.GetSlices([pl_record['name']])
              if not slices:
                   pointer = self.shell.AddSlice(pl_record)
+                  self.shell.SetSliceHrn(int(pointer), hrn)
              else:
                   pointer = slices[0]['slice_id']
  
                  can_add = ['first_name', 'last_name', 'title','email', 'password', 'phone', 'url', 'bio']
                  add_person_dict=dict ( [ (k,sfa_record[k]) for k in sfa_record if k in can_add ] )
                  pointer = self.shell.AddPerson(add_person_dict)
+                 self.shell.SetPersonHrn(int(pointer), hrn)
              else:
                  pointer = persons[0]['person_id']
      
              nodes = self.shell.GetNodes([pl_record['hostname']])
              if not nodes:
                  pointer = self.shell.AddNode(login_base, pl_record)
+                 self.shell.SetNodeHrn(int(pointer), hrn)
              else:
                  pointer = nodes[0]['node_id']
      
  
          if (type == "authority"):
              self.shell.UpdateSite(pointer, new_sfa_record)
+             self.shell.SetSiteHrn(pointer, hrn)
      
          elif type == "slice":
              pl_record=self.sfa_fields_to_pl_fields(type, hrn, new_sfa_record)
              if 'name' in pl_record:
                  pl_record.pop('name')
                  self.shell.UpdateSlice(pointer, pl_record)
+                 self.shell.SetSliceHrn(pointer, hrn)
      
          elif type == "user":
              # SMBAKER: UpdatePerson only allows a limited set of fields to be
              if 'email' in update_fields and not update_fields['email']:
                  del update_fields['email']
              self.shell.UpdatePerson(pointer, update_fields)
+             self.shell.SetPersonHrn(pointer, hrn)
      
              if new_key:
                  # must check this key against the previous one if it exists
          
          # get the registry records
          person_list, persons = [], {}
 -        person_list = dbsession.query (RegRecord).filter(RegRecord.pointer.in_(person_ids))
 +        person_list = self.api.dbsession().query (RegRecord).filter(RegRecord.pointer.in_(person_ids))
          # create a hrns keyed on the sfa record's pointer.
          # Its possible for multiple records to have the same pointer so
          # the dict's value will be a list of hrns.
          slices.handle_peer(None, None, persons, peer)
          # update sliver allocation states and set them to geni_provisioned
          sliver_ids = [sliver['sliver_id'] for sliver in slivers]
 -        SliverAllocation.set_allocations(sliver_ids, 'geni_provisioned')
 +        dbsession=self.api.dbsession()
 +        SliverAllocation.set_allocations(sliver_ids, 'geni_provisioned',dbsession)
          version_manager = VersionManager()
          rspec_version = version_manager.get_version(options['geni_rspec_version']) 
          return self.describe(urns, rspec_version, options=options)
                      self.shell.DeleteLeases(leases_ids)
       
                  # delete sliver allocation states
 -                SliverAllocation.delete_allocations(sliver_ids)
 +                dbsession=self.api.dbsession()
 +                SliverAllocation.delete_allocations(sliver_ids,dbsession)
              finally:
                  if peer:
                      self.shell.BindObjectToPeer('slice', slice_id, peer, slice['peer_slice_id'])
  
      # set the 'enabled' tag to 0
      def shutdown (self, xrn, options={}):
-         hrn = urn_to_hrn(xrn)
+         hrn, _ = urn_to_hrn(xrn)
          top_auth_hrn = top_auth(hrn)
-         if top_auth_hrn == self.hrn:
-             slicename = hrn_to_pl_slicename(hrn)
+         site_hrn = '.'.join(hrn.split('.')[:-1])
+         slice_part = hrn.split('.')[-1]
+         if top_auth_hrn == self.driver.hrn:
+             login_base = slice_hrn.split('.')[-2][:12]
          else:
-             slicename = xrn_to_ext_slicename(hrn)
+             login_base = hash_loginbase(site_hrn)
+         slicename = '_'.join([login_base, slice_part])
          slices = self.shell.GetSlices({'name': slicename}, ['slice_id'])
          if not slices:
              raise RecordNotFound(slice_hrn)
@@@ -8,8 -8,9 +8,8 @@@ from sfa.util.xrn import Xrn, get_leaf
  from sfa.rspecs.rspec import RSpec
  from sfa.planetlab.vlink import VLink
  from sfa.planetlab.topology import Topology
- from sfa.planetlab.plxrn import PlXrn, hrn_to_pl_slicename, xrn_to_hostname, xrn_to_ext_slicename, hrn_to_ext_loginbase, top_auth
+ from sfa.planetlab.plxrn import PlXrn, hrn_to_pl_slicename, xrn_to_hostname, top_auth, hash_loginbase
  from sfa.storage.model import SliverAllocation
 -from sfa.storage.alchemy import dbsession
  
  MAXINT =  2L**31-1
  
@@@ -169,11 -170,17 +169,17 @@@ class PlSlices
          for lease in rspec_requested_leases:
               requested_lease = {}
               slice_hrn, _ = urn_to_hrn(lease['slice_id'])
               top_auth_hrn = top_auth(slice_hrn)
+              site_hrn = '.'.join(slice_hrn.split('.')[:-1])
+              slice_part = slice_hrn.split('.')[-1]
               if top_auth_hrn == self.driver.hrn:
-                  slice_name = hrn_to_pl_slicename(lease['slice_id'])
+                  login_base = slice_hrn.split('.')[-2][:12]
               else:
-                  slice_name = xrn_to_ext_slicename(lease['slice_id'])
+                  login_base = hash_loginbase(site_hrn)
+              slice_name = '_'.join([login_base, slice_part])
               if slice_name != slice['name']:
                   continue
               elif Xrn(lease['component_id']).get_authority_urn().split(':')[0] != self.driver.hrn:
                                        component_id=component_id,
                                        slice_urn = slice_urn, 
                                        allocation_state='geni_allocated')      
 -            record.sync()
 +            record.sync(self.driver.api.dbsession())
          return resulting_nodes
  
      def free_egre_key(self):
      def verify_site(self, slice_xrn, slice_record={}, peer=None, sfa_peer=None, options={}):
          (slice_hrn, type) = urn_to_hrn(slice_xrn)
          top_auth_hrn = top_auth(slice_hrn)
+         site_hrn = '.'.join(slice_hrn.split('.')[:-1])
          if top_auth_hrn == self.driver.hrn:
-             # login base can't be longer than 20 characters
-             slicename = hrn_to_pl_slicename(slice_hrn)
-             authority_name = slicename.split('_')[0]
-             login_base = authority_name[:20]
+             login_base = slice_hrn.split('.')[-2][:12]
          else:
-             login_base = hrn_to_ext_loginbase(slice_hrn)
-             authority_name = login_base
+             login_base = hash_loginbase(site_hrn)
+         sites = self.driver.shell.GetSites({'peer_id': None},['site_id','name','abbreviated_name','login_base','hrn'])
+         # filter sites by hrn
+         site_exists = [site for site in sites if site['hrn'] == site_hrn]
  
-         sites = self.driver.shell.GetSites(login_base)
-         if not sites:
+         if not site_exists:
              # create new site record
-             site = {'name': 'sfa.%s' % authority_name,
-                     'abbreviated_name': authority_name,
+             site = {'name': 'sfa:%s' % site_hrn,
+                     'abbreviated_name': site_hrn,
                      'login_base': login_base,
                      'max_slices': 100,
                      'max_slivers': 1000,
                      'enabled': True,
                      'peer_site_id': None}
-             if peer:
-                 site['peer_site_id'] = slice_record.get('site_id', None)
              site['site_id'] = self.driver.shell.AddSite(site)
+             # Set site HRN
+             self.driver.shell.SetSiteHrn(int(site['site_id']), site_hrn)
+             # Tag this as created through SFA
+             self.driver.shell.SetSiteSfaCreated(int(site['site_id']), 'True')
              # exempt federated sites from monitor policies
-             self.driver.shell.AddSiteTag(site['site_id'], 'exempt_site_until', "20200101")
- #            # is this still necessary?
- #            # add record to the local registry 
- #            if sfa_peer and slice_record:
- #                peer_dict = {'type': 'authority', 'hrn': site_hrn, \
- #                             'peer_authority': sfa_peer, 'pointer': site['site_id']}
- #                self.registry.register_peer_object(self.credential, peer_dict)
+             self.driver.shell.AddSiteTag(int(site['site_id']), 'exempt_site_until', "20200101")
          else:
-             site =  sites[0]
-             if peer:
-                 # unbind from peer so we can modify if necessary. Will bind back later
-                 self.driver.shell.UnBindObjectFromPeer('site', site['site_id'], peer['shortname'])
+             site =  site_exists[0]
  
          return site
  
  
      def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer, expiration, options={}):
          top_auth_hrn = top_auth(slice_hrn)
+         site_hrn = '.'.join(slice_hrn.split('.')[:-1])
+         slice_part = slice_hrn.split('.')[-1]
          if top_auth_hrn == self.driver.hrn:
-             slicename = hrn_to_pl_slicename(slice_hrn)
-             parts = slicename.split("_")
-             login_base = parts[0]
+             login_base = slice_hrn.split('.')[-2][:12]
          else:
-             login_base = hrn_to_ext_loginbase(slice_hrn)
-             slicename = xrn_to_ext_slicename(slice_hrn)
+             login_base = hash_loginbase(site_hrn)
  
-         slices = self.driver.shell.GetSlices([slicename]) 
+         slice_name = '_'.join([login_base, slice_part])
+         slices = self.driver.shell.GetSlices({'peer_id': None},['slice_id','name','hrn'])
+         # Filter slices by HRN
+         slice_exists = [slice for slice in slices if slice['hrn'] == slice_hrn]
          expires = int(datetime_to_epoch(utcparse(expiration)))
-         if not slices:
-             slice = {'name': slicename,
-                      'url': slice_record.get('url', slice_hrn), 
+         if not slice_exists:
+             slice = {'name': slice_name,
+                      'url': slice_record.get('url', slice_hrn),
                       'description': slice_record.get('description', slice_hrn)}
              # add the slice                          
              slice['slice_id'] = self.driver.shell.AddSlice(slice)
-             slice['node_ids'] = []
-             slice['person_ids'] = []
              # set the slice HRN
-             self.driver.shell.SetSliceHrn(int(slice['slice_id']), slice_hrn)
-             if peer and slice_record:
-                 slice['peer_slice_id'] = slice_record.get('slice_id', None)
+             self.driver.shell.SetSliceHrn(int(slice['slice_id']), slice_hrn)       
+             # Tag this as created through SFA
+             self.driver.shell.SetSliceSfaCreated(int(slice['slice_id']), 'True')
              # set the expiration
-             self.driver.shell.UpdateSlice(slice['slice_id'], {'expires': expires}) 
+             self.driver.shell.UpdateSlice(int(slice['slice_id']), {'expires': expires})
          else:
-             slice = slices[0]
-             # Check slice HRN
-             if self.driver.shell.GetSliceHrn(slice['slice_id']) != slice_hrn:
-                 self.driver.shell.SetSliceHrn(slice['slice_id'], slice_hrn)
-             if peer and slice_record:
-                 slice['peer_slice_id'] = slice_record.get('slice_id', None)
-                 # unbind from peer so we can modify if necessary. Will bind back later
-                 self.driver.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname'])
-             
-               #Update expiration if necessary
-             if slice['expires'] != expires:
-                 self.driver.shell.UpdateSlice( slice['slice_id'], {'expires' : expires})
-        
-         return slice
+             slice = slice_exists[0]
+             #Update expiration if necessary
+             if slice.get('expires', None) != expires:
+                 self.driver.shell.UpdateSlice( int(slice['slice_id']), {'expires' : expires})
+         return self.driver.shell.GetSlices(int(slice['slice_id']))[0]
  
-     #def get_existing_persons(self, users):
      def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, options={}):
-         users_by_email = {}
-         users_by_site = defaultdict(list)
-         users_dict = {}
+         top_auth_hrn = top_auth(slice_hrn)
+         site_hrn = '.'.join(slice_hrn.split('.')[:-1])
+         slice_part = slice_hrn.split('.')[-1]
+         users_by_hrn = {}
          for user in users:
-             user['urn'] = user['urn'].lower()
-             hrn, type = urn_to_hrn(user['urn'])
-             username = get_leaf(hrn)
-             user['username'] = username
+            user['hrn'], _ = urn_to_hrn(user['urn'])
+            users_by_hrn[user['hrn']] = user
+         if top_auth_hrn == self.driver.hrn:
+             login_base = slice_hrn.split('.')[-2][:12]
+         else:
+             login_base = hash_loginbase(site_hrn)
  
-             top_auth_hrn = top_auth(hrn)
+         slice_name = '_'.join([login_base, slice_part])
  
-             if top_auth_hrn == self.driver.hrn:
-                 login_base = PlXrn(xrn=user['urn']).pl_login_base()
-             else:
-                 login_base = hrn_to_ext_loginbase(hrn)
+         persons = self.driver.shell.GetPersons({'peer_id': None},['person_id','email','hrn'])
+         site = self.driver.shell.GetSites({'peer_id': None, 'login_base': login_base})[0]
+         slice = self.driver.shell.GetSlices({'peer_id': None, 'name': slice_name})[0]
+         slice_persons = self.driver.shell.GetPersons({'peer_id': None, 'person_id': slice['person_ids']},['person_id','email','hrn'])
  
-             user['site'] = login_base
-             if 'email' in user:
-                 user['email'] = user['email'].lower()
-                 users_by_email[user['email']] = user
-                 users_dict[user['email']] = user
-             else:
-                 users_by_site[user['site']].append(user)
-         # start building a list of existing users
-         existing_user_ids = []
-         existing_user_ids_filter = []
-         if users_by_email:
-             existing_user_ids_filter.extend(users_by_email.keys())
-         if users_by_site:
-             for login_base in users_by_site:
-                 users = users_by_site[login_base]
-                 for user in users:
-                     existing_user_ids_filter.append(user['username']+'@geni.net') 
-         if existing_user_ids_filter:
-             # get existing users by email 
-             existing_users = self.driver.shell.GetPersons({'email': existing_user_ids_filter},
-                                                         ['person_id', 'key_ids', 'email'])
-             existing_user_ids.extend([user['email'] for user in existing_users])
-         if users_by_site:
-             # get a list of user sites (based on requeste user urns
-             site_list = self.driver.shell.GetSites(users_by_site.keys(), \
-                 ['site_id', 'login_base', 'person_ids'])
-             # get all existing users at these sites
-             sites = {}
-             site_user_ids = []
-             for site in site_list:
-                 sites[site['site_id']] = site
-                 site_user_ids.extend(site['person_ids'])
-             existing_site_persons_list = self.driver.shell.GetPersons(site_user_ids,
-                                                                     ['person_id', 'key_ids', 'email', 'site_ids'])
-             # all requested users are either existing users or new (added) users      
-             for login_base in users_by_site:
-                 requested_site_users = users_by_site[login_base]
-                 for requested_user in requested_site_users:
-                     user_found = False
-                     for existing_user in existing_site_persons_list:
-                         for site_id in existing_user['site_ids']:
-                             if site_id in sites:
-                                 site = sites[site_id]
-                                 if login_base == site['login_base'] and \
-                                    existing_user['email'].startswith(requested_user['username']+'@'):
-                                     existing_user_ids.append(existing_user['email'])
-                                     requested_user['email'] = existing_user['email']
-                                     users_dict[existing_user['email']] = requested_user
-                                     user_found = True
-                                     break
-                         if user_found:
-                             break
-                     if user_found == False:
-                         fake_email = requested_user['username'] + '@geni.net'
-                         requested_user['email'] = fake_email
-                         users_dict[fake_email] = requested_user
-         # requested slice users        
-         requested_user_ids = users_dict.keys()
-         # existing slice users
-         existing_slice_users_filter = {'person_id': slice_record.get('person_ids', [])}
-         existing_slice_users = self.driver.shell.GetPersons(existing_slice_users_filter,
-                                                           ['person_id', 'key_ids', 'email'])
-         existing_slice_user_ids = [user['email'] for user in existing_slice_users]
-         # users to be added, removed or updated
-         added_user_ids = set(requested_user_ids).difference(existing_user_ids)
-         added_slice_user_ids = set(requested_user_ids).difference(existing_slice_user_ids)
-         removed_user_ids = set(existing_slice_user_ids).difference(requested_user_ids)
-         updated_user_ids = set(existing_slice_user_ids).intersection(requested_user_ids)
-         # Remove stale users (only if we are not appending).
-         # Append by default.
-         append = options.get('append', True)
-         if append == False:
-             for removed_user_id in removed_user_ids:
-                 self.driver.shell.DeletePersonFromSlice(removed_user_id, slice_record['name'])
-         # update_existing users
-         updated_users_list = [user for user in users_dict.values() if user['email'] in \
-           updated_user_ids]
-         self.verify_keys(existing_slice_users, updated_users_list, peer, options)
-         added_persons = []
-         # add new users
-         for added_user_id in added_user_ids:
-             added_user = users_dict[added_user_id]
-             hrn, type = urn_to_hrn(added_user['urn'])
-             person = {
-                 'first_name': added_user.get('first_name', hrn),
-                 'last_name': added_user.get('last_name', hrn),
-                 'email': added_user_id,
-                 #'peer_person_id': None,
-                 #'keys': [],
-                 #'key_ids': added_user.get('key_ids', []),
-             }
-             person['person_id'] = self.driver.shell.AddPerson(person)
-             self.driver.shell.AddRoleToPerson('user', int(person['person_id']))
-             # check user HRN
-             if self.driver.shell.GetPersonHrn(int(person['person_id'])) != hrn:
-                 self.driver.shell.SetPersonHrn(int(person['person_id']), hrn)
+         persons_by_hrn = {}
+         persons_by_email = {}
+         for person in persons:
+            persons_by_hrn[person['hrn']] = person
+            persons_by_email[person['email']] = person
+         slice_persons_by_hrn = {}
+         for slice_person in slice_persons:
+            slice_persons_by_hrn[slice_person['hrn']] = slice_person
  
-             if peer:
-                 person['peer_person_id'] = added_user['person_id']
-             added_persons.append(person)
+         # sort persons by HRN
+         persons_to_add = set(users_by_hrn.keys()).difference(slice_persons_by_hrn.keys())
+         persons_to_delete = set(slice_persons_by_hrn.keys()).difference(users_by_hrn.keys())
+         persons_to_keep = set(users_by_hrn.keys()).intersection(slice_persons_by_hrn.keys())
  
-             # enable the account 
-             self.driver.shell.UpdatePerson(person['person_id'], {'enabled': True})
  
-             # add person to site
-             self.driver.shell.AddPersonToSite(added_user_id, added_user['site'])
+         persons_to_verify_keys = {}
  
-             for key_string in added_user.get('keys', []):
-                 key = {'key':key_string, 'key_type':'ssh'}
-                 key['key_id'] = self.driver.shell.AddPersonKey(person['person_id'], key)
-                 if 'keys' not in person:
-                     person['keys'] = []
-                 person['keys'].append(key)
+         # Add persons or add persons to slice
+         for person_hrn in persons_to_add:
+              person_email = users_by_hrn[person_hrn].get('email', None)
+              if person_email and person_email in persons_by_email.keys():
+                  # check if the user already exist in PL
+                  person_id = persons_by_email[person_email]['person_id']
+                  self.driver.shell.AddPersonToSlice(person_id, slice['slice_id'])
+                  persons_to_verify_keys[person_id] = users_by_hrn[person_hrn]
  
-             # add the registry record
- #            if sfa_peer:
- #                peer_dict = {'type': 'user', 'hrn': hrn, 'peer_authority': sfa_peer, \
- #                    'pointer': person['person_id']}
- #                self.registry.register_peer_object(self.credential, peer_dict)
+              else:
+                  person = {
+                            'first_name': person_hrn,
+                            'last_name': person_hrn,
+                            'email': users_by_hrn[person_hrn].get('email', "%s@geni.net"%person_hrn.split('.')[-1]),
+                           }
  
-         for added_slice_user_id in added_slice_user_ids.union(added_user_ids):
-             # add person to the slice 
-             self.driver.shell.AddPersonToSlice(added_slice_user_id, slice_record['name'])
-             # if this is a peer record then it should already be bound to a peer.
-             # no need to return worry about it getting bound later 
+                  person_id = self.driver.shell.AddPerson(person)
+                  self.driver.shell.AddRoleToPerson('user', int(person_id))
+                  # enable the account 
+                  self.driver.shell.UpdatePerson(int(person_id), {'enabled': True})
+                  self.driver.shell.SetPersonHrn(int(person_id), person_hrn)
+                  self.driver.shell.SetPersonSfaCreated(int(person_id), 'True')
+                  self.driver.shell.AddPersonToSite(int(person_id), site['site_id'])
+                  self.driver.shell.AddPersonToSlice(int(person_id), slice['slice_id'])
  
-         return added_persons
+                  # Add keys
+                  for key in users_by_hrn[person_hrn].get('keys', []):
+                       key = {'key':key, 'key_type':'ssh'}
+                       self.driver.shell.AddPersonKey(person_id, key)
  
  
-     def verify_keys(self, persons, users, peer, options={}):
-         # existing keys 
-         key_ids = []
-         for person in persons:
-             key_ids.extend(person['key_ids'])
-         keylist = self.driver.shell.GetKeys(key_ids, ['key_id', 'key'])
-         keydict = {}
-         for key in keylist:
-             keydict[key['key']] = key['key_id']
-         existing_keys = keydict.keys()
-         persondict = {}
-         for person in persons:
-             persondict[person['email']] = person
+         # Delete persons from slice     
+         for person_hrn in persons_to_delete:
+              person_id = slice_persons_by_hrn[person_hrn].get('person_id')
+              slice_id = slice['slice_id']
+              self.driver.shell.DeletePersonFromSlice(person_id, slice_id)
+         # Update kept persons
+         for person_hrn in persons_to_keep:
+              person_id = slice_persons_by_hrn[person_hrn].get('person_id')
+              persons_to_verify_keys[person_id] = users_by_hrn[person_hrn]
+         self.verify_keys(persons_to_verify_keys, peer, options)
+         return persons_to_add
+     def verify_keys(self, persons_to_verify_keys, peer, options={}):
+         # we only add keys that comes from sfa to persons in PL
+         for person_id in persons_to_verify_keys:
+              person_sfa_keys = persons_to_verify_keys[person_id].get('keys', [])
+              person_pl_keys = self.driver.shell.GetKeys({'person_id': int(person_id)})
+              person_pl_keys_list = [key['key'] for key in person_pl_keys]
+              keys_to_add = set(person_sfa_keys).difference(person_pl_keys_list)
+              for key_string in keys_to_add:
+                   key = {'key': key_string, 'key_type': 'ssh'}
+                   self.driver.shell.AddPersonKey(int(person_id), key)
  
-         # add new keys
-         requested_keys = []
-         updated_persons = []
-         for user in users:
-             user_keys = user.get('keys', [])
-             updated_persons.append(user)
-             for key_string in user_keys:
-                 requested_keys.append(key_string)
-                 if key_string not in existing_keys:
-                     key = {'key': key_string, 'key_type': 'ssh'}
-                     try:
-                         if peer:
-                             person = persondict[user['email']]
-                             self.driver.shell.UnBindObjectFromPeer('person', person['person_id'], peer['shortname'])
-                         key['key_id'] = self.driver.shell.AddPersonKey(user['email'], key)
-                         if peer:
-                             key_index = user_keys.index(key['key'])
-                             remote_key_id = user['key_ids'][key_index]
-                             self.driver.shell.BindObjectToPeer('key', key['key_id'], peer['shortname'], remote_key_id)
-                     finally:
-                         if peer:
-                             self.driver.shell.BindObjectToPeer('person', person['person_id'], peer['shortname'], user['person_id'])
-         # remove old keys (only if we are not appending)
-         append = options.get('append', True)
-         if append == False:
-             removed_keys = set(existing_keys).difference(requested_keys)
-             for existing_key_id in keydict:
-                 if keydict[existing_key_id] in removed_keys:
-                     try:
-                         if peer:
-                             self.driver.shell.UnBindObjectFromPeer('key', existing_key_id, peer['shortname'])
-                         self.driver.shell.DeleteKey(existing_key_id)
-                     except:
-                         pass
  
      def verify_slice_attributes(self, slice, requested_slice_attributes, options={}, admin=False):
          append = options.get('append', True)
          # get sliver attributes
          added_slice_attributes = []
          removed_slice_attributes = []
-         ignored_slice_attribute_names = []
+         # we need to keep the slice hrn anyway
+         ignored_slice_attribute_names = ['hrn']
          existing_slice_attributes = self.driver.shell.GetSliceTags({'slice_id': slice['slice_id']})
  
          # get attributes that should be removed
                  # If a slice already has a admin only role it was probably given to them by an
                  # admin, so we should ignore it.
                  ignored_slice_attribute_names.append(slice_tag['tagname'])
+                 attribute_found=True
              else:
                  # If an existing slice attribute was not found in the request it should
                  # be removed