sfa/importer/plimporter.py

   1 import os
   2
   3 from sfa.util.config import Config
   4 from sfa.util.xrn import Xrn, get_leaf, get_authority, hrn_to_urn
   5 from sfa.util.plxrn import hostname_to_hrn, slicename_to_hrn, email_to_hrn, hrn_to_pl_slicename
   6
   7 from sfa.trust.gid import create_uuid
   8 from sfa.trust.certificate import convert_public_key, Keypair
   9
  10 from sfa.storage.alchemy import dbsession
  11 from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, RegUser, RegKey
  12
  13 from sfa.plc.plshell import PlShell
  14
  15 def _get_site_hrn(interface_hrn, site):
  16     # Hardcode 'internet2' into the hrn for sites hosting
  17     # internet2 nodes. This is a special operation for some vini
  18     # sites only
  19     hrn = ".".join([interface_hrn, site['login_base']])
  20     if ".vini" in interface_hrn and interface_hrn.endswith('vini'):
  21         if site['login_base'].startswith("i2") or site['login_base'].startswith("nlr"):
  22             hrn = ".".join([interface_hrn, "internet2", site['login_base']])
  23     return hrn
  24
  25
  26 class PlImporter:
  27
  28     def __init__ (self, auth_hierarchy, logger):
  29         self.auth_hierarchy = auth_hierarchy
  30         self.logger=logger
  31
  32     def record_options (self, parser):
  33         self.logger.debug ("PlImporter no options yet")
  34         pass
  35
  36     def run (self, options):
  37         # we don't have any options for now
  38         config = Config ()
  39         interface_hrn = config.SFA_INTERFACE_HRN
  40         root_auth = config.SFA_REGISTRY_ROOT_AUTH
  41         shell = PlShell (config)
  42
  43         # create dict of all existing sfa records
  44         records_by_hrn_type = {}
  45         records_by_type_pointer = {}
  46         key_ids = []
  47         records = dbsession.query(RegRecord)
  48         for record in records:
  49             records_by_hrn_type[ (record.hrn, record.type,) ] = record
  50             if record.pointer != -1:
  51                 records_by_type_pointer [ (record.type, record.pointer,) ] = record
  52
  53         # Get all plc sites
  54         # retrieve only required stuf
  55         sites = shell.GetSites({'peer_id': None, 'enabled' : True},
  56                                ['site_id','login_base','node_ids','slice_ids','person_ids',])
  57         sites_dict = {}
  58         for site in sites:
  59             sites_dict[site['login_base']] = site
  60
  61         # Get all plc users
  62         persons = shell.GetPersons({'peer_id': None, 'enabled': True},
  63                                    ['person_id', 'email', 'key_ids', 'site_ids'])
  64         persons_dict = {}
  65         for person in persons:
  66             persons_dict[person['person_id']] = person
  67             key_ids.extend(person['key_ids'])
  68
  69         # Get all plc public keys
  70         keys = shell.GetKeys( {'peer_id': None, 'key_id': key_ids})
  71         keys_by_id = {}
  72         for key in keys: keys_by_id[key['key_id']] = key
  73
  74         # create a dict person_id -> [ (plc)keys ]
  75         keys_by_person_id = {}
  76         for person in persons:
  77             pubkeys = []
  78             for key_id in person['key_ids']:
  79                 pubkeys.append(keys_by_id[key_id])
  80             keys_by_person_id[person['person_id']] = pubkeys
  81
  82         # Get all plc nodes
  83         nodes = shell.GetNodes( {'peer_id': None}, ['node_id', 'hostname', 'site_id'])
  84         nodes_dict = {}
  85         for node in nodes:
  86             nodes_dict[node['node_id']] = node
  87
  88         # Get all plc slices
  89         slices = shell.GetSlices( {'peer_id': None}, ['slice_id', 'name'])
  90         slices_dict = {}
  91         for slice in slices:
  92             slices_dict[slice['slice_id']] = slice
  93
  94         # special case for vini
  95         if ".vini" in interface_hrn and interface_hrn.endswith('vini'):
  96             # create a fake internet2 site first
  97             i2site = {'name': 'Internet2', 'login_base': 'internet2', 'site_id': -1}
  98             site_hrn = _get_site_hrn(interface_hrn, i2site)
  99             # import if hrn is not in list of existing hrns or if the hrn exists
 100             # but its not a site record
 101             if (site_hrn, 'authority') not in records_by_hrn_type:
 102                 urn = hrn_to_urn(site_hrn, 'authority')
 103                 if not self.auth_hierarchy.auth_exists(urn):
 104                     self.auth_hierarchy.create_auth(urn)
 105                 auth_info = self.auth_hierarchy.get_auth_info(urn)
 106                 auth_record = RegAuthority()
 107                 auth_record.type='authority'
 108                 auth_record.hrn=site_hrn
 109                 auth_record.gid=auth_info.get_gid_object()
 110                 auth_record.pointer=site['site_id']
 111                 auth_record.authority=get_authority(site_hrn)
 112                 dbsession.add(auth_record)
 113                 dbsession.commit()
 114                 self.logger.info("PlImporter: Imported authority (vini site) %s"%auth_record)
 115
 116         # start importing
 117         for site in sites:
 118             site_hrn = _get_site_hrn(interface_hrn, site)
 119
 120             # import if hrn is not in list of existing hrns or if the hrn exists
 121             # but its not a site record
 122             if (site_hrn, 'authority') not in records_by_hrn_type:
 123                 try:
 124                     urn = hrn_to_urn(site_hrn, 'authority')
 125                     if not self.auth_hierarchy.auth_exists(urn):
 126                         self.auth_hierarchy.create_auth(urn)
 127                     auth_info = self.auth_hierarchy.get_auth_info(urn)
 128                     auth_record = RegAuthority()
 129                     auth_record.type='authority'
 130                     auth_record.hrn=site_hrn
 131                     auth_record.gid=auth_info.get_gid_object()
 132                     auth_record.pointer=site['site_id']
 133                     auth_record.authority=get_authority(site_hrn)
 134                     dbsession.add(auth_record)
 135                     dbsession.commit()
 136                     self.logger.info("PlImporter: imported authority (site) : %s" % auth_record)
 137                 except:
 138                     # if the site import fails then there is no point in trying to import the
 139                     # site's child records (node, slices, persons), so skip them.
 140                     self.logger.log_exc("PlImporter: failed to import site. Skipping child records")
 141                     continue
 142
 143             # import node records
 144             for node_id in site['node_ids']:
 145                 if node_id not in nodes_dict:
 146                     continue
 147                 node = nodes_dict[node_id]
 148                 site_auth = get_authority(site_hrn)
 149                 site_name = get_leaf(site_hrn)
 150                 hrn =  hostname_to_hrn(site_auth, site_name, node['hostname'])
 151                 if len(hrn) > 64:
 152                     hrn = hrn[:64]
 153                 if (hrn, 'node') not in records_by_hrn_type:
 154                     try:
 155                         pkey = Keypair(create=True)
 156                         urn = hrn_to_urn(hrn, 'node')
 157                         node_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
 158                         node_record = RegNode ()
 159                         node_record.type='node'
 160                         node_record.hrn=hrn
 161                         node_record.gid=node_gid
 162                         node_record.pointer =node['node_id']
 163                         node_record.authority=get_authority(hrn)
 164                         dbsession.add(node_record)
 165                         dbsession.commit()
 166                         self.logger.info("PlImporter: imported node: %s" % node_record)
 167                     except:
 168                         self.logger.log_exc("PlImporter: failed to import node")
 169
 170
 171             # import slices
 172             for slice_id in site['slice_ids']:
 173                 if slice_id not in slices_dict:
 174                     continue
 175                 slice = slices_dict[slice_id]
 176                 hrn = slicename_to_hrn(interface_hrn, slice['name'])
 177                 if (hrn, 'slice') not in records_by_hrn_type:
 178                     try:
 179                         pkey = Keypair(create=True)
 180                         urn = hrn_to_urn(hrn, 'slice')
 181                         slice_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
 182                         slice_record = RegSlice ()
 183                         slice_record.type='slice'
 184                         slice_record.hrn=hrn
 185                         slice_record.gid=slice_gid
 186                         slice_record.pointer=slice['slice_id']
 187                         slice_record.authority=get_authority(hrn)
 188                         dbsession.add(slice_record)
 189                         dbsession.commit()
 190                         self.logger.info("PlImporter: imported slice: %s" % slice_record)
 191                     except:
 192                         self.logger.log_exc("PlImporter: failed to  import slice")
 193
 194             # import persons
 195             for person_id in site['person_ids']:
 196                 if person_id not in persons_dict:
 197                     self.logger.warning ("PlImporter: skipping person %s"%person_id)
 198                     continue
 199                 person = persons_dict[person_id]
 200                 hrn = email_to_hrn(site_hrn, person['email'])
 201                 if len(hrn) > 64:
 202                     hrn = hrn[:64]
 203
 204                 previous_record = records_by_hrn_type.get( (hrn, 'user',) )
 205                 if not previous_record:
 206                     previous_record = records_by_type_pointer.get ( ('user', person_id,) )
 207                 # if user's primary key has changed then we need to update the
 208                 # users gid by forcing an update here
 209                 plc_keys = []
 210                 sfa_keys = []
 211                 if previous_record:
 212                     sfa_keys = previous_record.keys
 213                 if person_id in keys_by_person_id:
 214                     plc_keys = keys_by_person_id[person_id]
 215                 update_record = False
 216                 def key_in_list (key,sfa_keys):
 217                     for reg_key in sfa_keys:
 218                         if reg_key.key==key['key']: return True
 219                     return False
 220                 for key in plc_keys:
 221                     if not key_in_list (key,sfa_keys):
 222                         update_record = True
 223
 224                 if not previous_record or update_record:
 225                     try:
 226                         pubkey=None
 227                         if 'key_ids' in person and person['key_ids']:
 228                             # randomly pick first key in set
 229                             pubkey = plc_keys[0]
 230                             try:
 231                                 pkey = convert_public_key(pubkey['key'])
 232                             except:
 233                                 self.logger.warn('PlImporter: unable to convert public key for %s' % hrn)
 234                                 pkey = Keypair(create=True)
 235                         else:
 236                             # the user has no keys. Creating a random keypair for the user's gid
 237                             self.logger.warn("PlImporter: person %s does not have a PL public key"%hrn)
 238                             pkey = Keypair(create=True)
 239                         urn = hrn_to_urn(hrn, 'user')
 240                         person_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
 241                         if previous_record:
 242                             previous_record.gid=person_gid
 243                             if pubkey: previous_record.keys=[RegKey (pubkey['key'], pubkey['key_id'])]
 244                             self.logger.info("PlImporter: updated person: %s" % previous_record)
 245                         else:
 246                             new_record = RegUser (hrn=hrn, gid=person_gid,
 247                                                   pointer=person['person_id'],
 248                                                   authority=get_authority(hrn),
 249                                                   email=person['email'])
 250                             if pubkey: new_record.keys=[RegKey (pubkey['key'], pubkey['key_id'])]
 251                             dbsession.add (new_record)
 252                             dbsession.commit()
 253                             self.logger.info("PlImporter: imported person: %s" % new_record)
 254                     except:
 255                         self.logger.log_exc("PlImporter: failed to import person.")
 256
 257         # remove stale records
 258         system_records = [interface_hrn, root_auth, interface_hrn + '.slicemanager']
 259         for record in records:
 260             record_hrn=record.hrn
 261             if record_hrn in system_records:
 262                 continue
 263             if record.peer_authority:
 264                 continue
 265             type=record.type
 266             hrn=record.hrn
 267             # dont delete vini's internet2 placeholdder record
 268             # normally this would be deleted becuase it does not have a plc record
 269             if ".vini" in interface_hrn and interface_hrn.endswith('vini') and \
 270                record_hrn.endswith("internet2"):
 271                 continue
 272
 273             found = False
 274
 275             if isinstance (record, RegAuthority):
 276                 for site in sites:
 277                     site_hrn = interface_hrn + "." + site['login_base']
 278                     if site_hrn == record_hrn and site['site_id'] == record.pointer:
 279                         found = True
 280                         break
 281
 282             elif isinstance (record, RegUser):
 283                 login_base = get_leaf(get_authority(record_hrn))
 284                 username = get_leaf(record_hrn)
 285                 if login_base in sites_dict:
 286                     site = sites_dict[login_base]
 287                     for person in persons:
 288                         tmp_username = person['email'].split("@")[0]
 289                         alt_username = person['email'].split("@")[0].replace(".", "_").replace("+", "_")
 290                         if username in [tmp_username, alt_username] and \
 291                            site['site_id'] in person['site_ids'] and \
 292                            person['person_id'] == record.pointer:
 293                             found = True
 294                             break
 295
 296             elif isinstance (record, RegSlice):
 297                 slicename = hrn_to_pl_slicename(record_hrn)
 298                 for slice in slices:
 299                     if slicename == slice['name'] and \
 300                        slice['slice_id'] == record.pointer:
 301                         found = True
 302                         break
 303
 304             elif isinstance (record, RegNode):
 305                 login_base = get_leaf(get_authority(record_hrn))
 306                 nodename = Xrn.unescape(get_leaf(record_hrn))
 307                 if login_base in sites_dict:
 308                     site = sites_dict[login_base]
 309                     for node in nodes:
 310                         tmp_nodename = node['hostname']
 311                         if tmp_nodename == nodename and \
 312                            node['site_id'] == site['site_id'] and \
 313                            node['node_id'] == record.pointer:
 314                             found = True
 315                             break
 316             else:
 317                 continue
 318
 319             if not found:
 320                 try:
 321                     record_object = records_by_hrn_type[(record_hrn, type)]
 322                     self.logger.info("PlImporter: deleting record: %s" % record)
 323                     dbsession.delete(record_object)
 324                     dbsession.commit()
 325                 except:
 326                     self.logger.log_exc("PlImporter: failded to delete record")
 327
 328 #        # save pub keys
 329 #        self.logger.info('Import: saving current pub keys')
 330 #        save_keys(keys_filename, person_keys)
 331