sfa/plc/plcsfaapi.py

   1 import xmlrpclib
   2 #
   3 from sfa.util.faults import MissingSfaInfo
   4 from sfa.util.sfalogging import logger
   5 from sfa.util.table import SfaTable
   6
   7 from sfa.util.xrn import hrn_to_urn
   8 from sfa.util.plxrn import slicename_to_hrn, hostname_to_hrn, hrn_to_pl_slicename, hrn_to_pl_login_base
   9
  10 from sfa.server.sfaapi import SfaApi
  11
  12 #################### xxx should move into util/defaultdict
  13 try:
  14     from collections import defaultdict
  15 except:
  16     class defaultdict(dict):
  17         def __init__(self, default_factory=None, *a, **kw):
  18             if (default_factory is not None and
  19                 not hasattr(default_factory, '__call__')):
  20                 raise TypeError('first argument must be callable')
  21             dict.__init__(self, *a, **kw)
  22             self.default_factory = default_factory
  23         def __getitem__(self, key):
  24             try:
  25                 return dict.__getitem__(self, key)
  26             except KeyError:
  27                 return self.__missing__(key)
  28         def __missing__(self, key):
  29             if self.default_factory is None:
  30                 raise KeyError(key)
  31             self[key] = value = self.default_factory()
  32             return value
  33         def __reduce__(self):
  34             if self.default_factory is None:
  35                 args = tuple()
  36             else:
  37                 args = self.default_factory,
  38             return type(self), args, None, None, self.items()
  39         def copy(self):
  40             return self.__copy__()
  41         def __copy__(self):
  42             return type(self)(self.default_factory, self)
  43         def __deepcopy__(self, memo):
  44             import copy
  45             return type(self)(self.default_factory,
  46                               copy.deepcopy(self.items()))
  47         def __repr__(self):
  48             return 'defaultdict(%s, %s)' % (self.default_factory,
  49                                             dict.__repr__(self))
  50 ## end of http://code.activestate.com/recipes/523034/ }}}
  51
  52 def list_to_dict(recs, key):
  53     """
  54     convert a list of dictionaries into a dictionary keyed on the
  55     specified dictionary key
  56     """
  57     keys = [rec[key] for rec in recs]
  58     return dict(zip(keys, recs))
  59
  60 class PlcSfaApi(SfaApi):
  61
  62     def __init__ (self, encoding="utf-8", methods='sfa.methods',
  63                   config = "/etc/sfa/sfa_config.py",
  64                   peer_cert = None, interface = None,
  65                   key_file = None, cert_file = None, cache = None):
  66         SfaApi.__init__(self, encoding=encoding, methods=methods,
  67                         config=config,
  68                         peer_cert=peer_cert, interface=interface,
  69                         key_file=key_file,
  70                         cert_file=cert_file, cache=cache)
  71
  72         self.SfaTable = SfaTable
  73         # Initialize the PLC shell only if SFA wraps a myPLC
  74         rspec_type = self.config.get_aggregate_type()
  75         if (rspec_type == 'pl' or rspec_type == 'vini' or \
  76             rspec_type == 'eucalyptus' or rspec_type == 'max'):
  77             self.plshell = self.getPLCShell()
  78             self.plshell_version = "4.3"
  79
  80     def getPLCShell(self):
  81         self.plauth = {'Username': self.config.SFA_PLC_USER,
  82                        'AuthMethod': 'password',
  83                        'AuthString': self.config.SFA_PLC_PASSWORD}
  84
  85         # The native shell (PLC.Shell.Shell) is more efficient than xmlrpc,
  86         # but it leaves idle db connections open. use xmlrpc until we can figure
  87         # out why PLC.Shell.Shell doesn't close db connection properly
  88         #try:
  89         #    sys.path.append(os.path.dirname(os.path.realpath("/usr/bin/plcsh")))
  90         #    self.plshell_type = 'direct'
  91         #    import PLC.Shell
  92         #    shell = PLC.Shell.Shell(globals = globals())
  93         #except:
  94
  95         self.plshell_type = 'xmlrpc'
  96         url = self.config.SFA_PLC_URL
  97         shell = xmlrpclib.Server(url, verbose = 0, allow_none = True)
  98         return shell
  99
 100     ##
 101     # Convert SFA fields to PLC fields for use when registering up updating
 102     # registry record in the PLC database
 103     #
 104     # @param type type of record (user, slice, ...)
 105     # @param hrn human readable name
 106     # @param sfa_fields dictionary of SFA fields
 107     # @param pl_fields dictionary of PLC fields (output)
 108
 109     def sfa_fields_to_pl_fields(self, type, hrn, record):
 110
 111         def convert_ints(tmpdict, int_fields):
 112             for field in int_fields:
 113                 if field in tmpdict:
 114                     tmpdict[field] = int(tmpdict[field])
 115
 116         pl_record = {}
 117         #for field in record:
 118         #    pl_record[field] = record[field]
 119
 120         if type == "slice":
 121             if not "instantiation" in pl_record:
 122                 pl_record["instantiation"] = "plc-instantiated"
 123             pl_record["name"] = hrn_to_pl_slicename(hrn)
 124             if "url" in record:
 125                pl_record["url"] = record["url"]
 126             if "description" in record:
 127                 pl_record["description"] = record["description"]
 128             if "expires" in record:
 129                 pl_record["expires"] = int(record["expires"])
 130
 131         elif type == "node":
 132             if not "hostname" in pl_record:
 133                 if not "hostname" in record:
 134                     raise MissingSfaInfo("hostname")
 135                 pl_record["hostname"] = record["hostname"]
 136             if not "model" in pl_record:
 137                 pl_record["model"] = "geni"
 138
 139         elif type == "authority":
 140             pl_record["login_base"] = hrn_to_pl_login_base(hrn)
 141
 142             if not "name" in pl_record:
 143                 pl_record["name"] = hrn
 144
 145             if not "abbreviated_name" in pl_record:
 146                 pl_record["abbreviated_name"] = hrn
 147
 148             if not "enabled" in pl_record:
 149                 pl_record["enabled"] = True
 150
 151             if not "is_public" in pl_record:
 152                 pl_record["is_public"] = True
 153
 154         return pl_record
 155
 156     def fill_record_pl_info(self, records):
 157         """
 158         Fill in the planetlab specific fields of a SFA record. This
 159         involves calling the appropriate PLC method to retrieve the
 160         database record for the object.
 161
 162         PLC data is filled into the pl_info field of the record.
 163
 164         @param record: record to fill in field (in/out param)
 165         """
 166         # get ids by type
 167         node_ids, site_ids, slice_ids = [], [], []
 168         person_ids, key_ids = [], []
 169         type_map = {'node': node_ids, 'authority': site_ids,
 170                     'slice': slice_ids, 'user': person_ids}
 171
 172         for record in records:
 173             for type in type_map:
 174                 if type == record['type']:
 175                     type_map[type].append(record['pointer'])
 176
 177         # get pl records
 178         nodes, sites, slices, persons, keys = {}, {}, {}, {}, {}
 179         if node_ids:
 180             node_list = self.plshell.GetNodes(self.plauth, node_ids)
 181             nodes = list_to_dict(node_list, 'node_id')
 182         if site_ids:
 183             site_list = self.plshell.GetSites(self.plauth, site_ids)
 184             sites = list_to_dict(site_list, 'site_id')
 185         if slice_ids:
 186             slice_list = self.plshell.GetSlices(self.plauth, slice_ids)
 187             slices = list_to_dict(slice_list, 'slice_id')
 188         if person_ids:
 189             person_list = self.plshell.GetPersons(self.plauth, person_ids)
 190             persons = list_to_dict(person_list, 'person_id')
 191             for person in persons:
 192                 key_ids.extend(persons[person]['key_ids'])
 193
 194         pl_records = {'node': nodes, 'authority': sites,
 195                       'slice': slices, 'user': persons}
 196
 197         if key_ids:
 198             key_list = self.plshell.GetKeys(self.plauth, key_ids)
 199             keys = list_to_dict(key_list, 'key_id')
 200
 201         # fill record info
 202         for record in records:
 203             # records with pointer==-1 do not have plc info.
 204             # for example, the top level authority records which are
 205             # authorities, but not PL "sites"
 206             if record['pointer'] == -1:
 207                 continue
 208
 209             for type in pl_records:
 210                 if record['type'] == type:
 211                     if record['pointer'] in pl_records[type]:
 212                         record.update(pl_records[type][record['pointer']])
 213                         break
 214             # fill in key info
 215             if record['type'] == 'user':
 216                 if 'key_ids' not in record:
 217                     logger.info("user record has no 'key_ids' - need to import from myplc ?")
 218                 else:
 219                     pubkeys = [keys[key_id]['key'] for key_id in record['key_ids'] if key_id in keys]
 220                     record['keys'] = pubkeys
 221
 222         # fill in record hrns
 223         records = self.fill_record_hrns(records)
 224
 225         return records
 226
 227     def fill_record_hrns(self, records):
 228         """
 229         convert pl ids to hrns
 230         """
 231
 232         # get ids
 233         slice_ids, person_ids, site_ids, node_ids = [], [], [], []
 234         for record in records:
 235             if 'site_id' in record:
 236                 site_ids.append(record['site_id'])
 237             if 'site_ids' in record:
 238                 site_ids.extend(record['site_ids'])
 239             if 'person_ids' in record:
 240                 person_ids.extend(record['person_ids'])
 241             if 'slice_ids' in record:
 242                 slice_ids.extend(record['slice_ids'])
 243             if 'node_ids' in record:
 244                 node_ids.extend(record['node_ids'])
 245
 246         # get pl records
 247         slices, persons, sites, nodes = {}, {}, {}, {}
 248         if site_ids:
 249             site_list = self.plshell.GetSites(self.plauth, site_ids, ['site_id', 'login_base'])
 250             sites = list_to_dict(site_list, 'site_id')
 251         if person_ids:
 252             person_list = self.plshell.GetPersons(self.plauth, person_ids, ['person_id', 'email'])
 253             persons = list_to_dict(person_list, 'person_id')
 254         if slice_ids:
 255             slice_list = self.plshell.GetSlices(self.plauth, slice_ids, ['slice_id', 'name'])
 256             slices = list_to_dict(slice_list, 'slice_id')
 257         if node_ids:
 258             node_list = self.plshell.GetNodes(self.plauth, node_ids, ['node_id', 'hostname'])
 259             nodes = list_to_dict(node_list, 'node_id')
 260
 261         # convert ids to hrns
 262         for record in records:
 263             # get all relevant data
 264             type = record['type']
 265             pointer = record['pointer']
 266             auth_hrn = self.hrn
 267             login_base = ''
 268             if pointer == -1:
 269                 continue
 270
 271             if 'site_id' in record:
 272                 site = sites[record['site_id']]
 273                 login_base = site['login_base']
 274                 record['site'] = ".".join([auth_hrn, login_base])
 275             if 'person_ids' in record:
 276                 emails = [persons[person_id]['email'] for person_id in record['person_ids'] \
 277                           if person_id in  persons]
 278                 usernames = [email.split('@')[0] for email in emails]
 279                 person_hrns = [".".join([auth_hrn, login_base, username]) for username in usernames]
 280                 record['persons'] = person_hrns
 281             if 'slice_ids' in record:
 282                 slicenames = [slices[slice_id]['name'] for slice_id in record['slice_ids'] \
 283                               if slice_id in slices]
 284                 slice_hrns = [slicename_to_hrn(auth_hrn, slicename) for slicename in slicenames]
 285                 record['slices'] = slice_hrns
 286             if 'node_ids' in record:
 287                 hostnames = [nodes[node_id]['hostname'] for node_id in record['node_ids'] \
 288                              if node_id in nodes]
 289                 node_hrns = [hostname_to_hrn(auth_hrn, login_base, hostname) for hostname in hostnames]
 290                 record['nodes'] = node_hrns
 291             if 'site_ids' in record:
 292                 login_bases = [sites[site_id]['login_base'] for site_id in record['site_ids'] \
 293                                if site_id in sites]
 294                 site_hrns = [".".join([auth_hrn, lbase]) for lbase in login_bases]
 295                 record['sites'] = site_hrns
 296
 297         return records
 298
 299     def fill_record_sfa_info(self, records):
 300
 301         def startswith(prefix, values):
 302             return [value for value in values if value.startswith(prefix)]
 303
 304         # get person ids
 305         person_ids = []
 306         site_ids = []
 307         for record in records:
 308             person_ids.extend(record.get("person_ids", []))
 309             site_ids.extend(record.get("site_ids", []))
 310             if 'site_id' in record:
 311                 site_ids.append(record['site_id'])
 312
 313         # get all pis from the sites we've encountered
 314         # and store them in a dictionary keyed on site_id
 315         site_pis = {}
 316         if site_ids:
 317             pi_filter = {'|roles': ['pi'], '|site_ids': site_ids}
 318             pi_list = self.plshell.GetPersons(self.plauth, pi_filter, ['person_id', 'site_ids'])
 319             for pi in pi_list:
 320                 # we will need the pi's hrns also
 321                 person_ids.append(pi['person_id'])
 322
 323                 # we also need to keep track of the sites these pis
 324                 # belong to
 325                 for site_id in pi['site_ids']:
 326                     if site_id in site_pis:
 327                         site_pis[site_id].append(pi)
 328                     else:
 329                         site_pis[site_id] = [pi]
 330
 331         # get sfa records for all records associated with these records.
 332         # we'll replace pl ids (person_ids) with hrns from the sfa records
 333         # we obtain
 334
 335         # get the sfa records
 336         table = self.SfaTable()
 337         person_list, persons = [], {}
 338         person_list = table.find({'type': 'user', 'pointer': person_ids})
 339         # create a hrns keyed on the sfa record's pointer.
 340         # Its possible for  multiple records to have the same pointer so
 341         # the dict's value will be a list of hrns.
 342         persons = defaultdict(list)
 343         for person in person_list:
 344             persons[person['pointer']].append(person)
 345
 346         # get the pl records
 347         pl_person_list, pl_persons = [], {}
 348         pl_person_list = self.plshell.GetPersons(self.plauth, person_ids, ['person_id', 'roles'])
 349         pl_persons = list_to_dict(pl_person_list, 'person_id')
 350
 351         # fill sfa info
 352         for record in records:
 353             # skip records with no pl info (top level authorities)
 354             #if record['pointer'] == -1:
 355             #    continue
 356             sfa_info = {}
 357             type = record['type']
 358             if (type == "slice"):
 359                 # all slice users are researchers
 360                 record['geni_urn'] = hrn_to_urn(record['hrn'], 'slice')
 361                 record['PI'] = []
 362                 record['researcher'] = []
 363                 for person_id in record.get('person_ids', []):
 364                     hrns = [person['hrn'] for person in persons[person_id]]
 365                     record['researcher'].extend(hrns)
 366
 367                 # pis at the slice's site
 368                 if 'site_id' in record and record['site_id'] in site_pis:
 369                     pl_pis = site_pis[record['site_id']]
 370                     pi_ids = [pi['person_id'] for pi in pl_pis]
 371                     for person_id in pi_ids:
 372                         hrns = [person['hrn'] for person in persons[person_id]]
 373                         record['PI'].extend(hrns)
 374                         record['geni_creator'] = record['PI']
 375
 376             elif (type.startswith("authority")):
 377                 record['url'] = None
 378                 if record['hrn'] in self.aggregates:
 379
 380                     record['url'] = self.aggregates[record['hrn']].get_url()
 381
 382                 if record['pointer'] != -1:
 383                     record['PI'] = []
 384                     record['operator'] = []
 385                     record['owner'] = []
 386                     for pointer in record.get('person_ids', []):
 387                         if pointer not in persons or pointer not in pl_persons:
 388                             # this means there is not sfa or pl record for this user
 389                             continue
 390                         hrns = [person['hrn'] for person in persons[pointer]]
 391                         roles = pl_persons[pointer]['roles']
 392                         if 'pi' in roles:
 393                             record['PI'].extend(hrns)
 394                         if 'tech' in roles:
 395                             record['operator'].extend(hrns)
 396                         if 'admin' in roles:
 397                             record['owner'].extend(hrns)
 398                         # xxx TODO: OrganizationName
 399             elif (type == "node"):
 400                 sfa_info['dns'] = record.get("hostname", "")
 401                 # xxx TODO: URI, LatLong, IP, DNS
 402
 403             elif (type == "user"):
 404                 sfa_info['email'] = record.get("email", "")
 405                 sfa_info['geni_urn'] = hrn_to_urn(record['hrn'], 'user')
 406                 sfa_info['geni_certificate'] = record['gid']
 407                 # xxx TODO: PostalAddress, Phone
 408             record.update(sfa_info)
 409
 410     def fill_record_info(self, records):
 411         """
 412         Given a SFA record, fill in the PLC specific and SFA specific
 413         fields in the record.
 414         """
 415         if not isinstance(records, list):
 416             records = [records]
 417
 418         self.fill_record_pl_info(records)
 419         self.fill_record_sfa_info(records)
 420
 421     def update_membership_list(self, oldRecord, record, listName, addFunc, delFunc):
 422         # get a list of the HRNs that are members of the old and new records
 423         if oldRecord:
 424             oldList = oldRecord.get(listName, [])
 425         else:
 426             oldList = []
 427         newList = record.get(listName, [])
 428         # xxx ugly hack - somehow we receive here a list of {'text':value}
 429         # instead of an expected list of strings
 430         # please remove once this is issue is cleanly fixed
 431         def normalize (value):
 432             from types import StringTypes
 433             if isinstance(value,StringTypes): return value
 434             elif isinstance(value,dict):
 435                 newvalue=value['text']
 436                 logger.info("Normalizing %s=>%s"%(value,newvalue))
 437                 return newvalue
 438         newList=[normalize(v) for v in newList]
 439
 440         # if the lists are the same, then we don't have to update anything
 441         if (oldList == newList):
 442             return
 443
 444         # build a list of the new person ids, by looking up each person to get
 445         # their pointer
 446         newIdList = []
 447         table = self.SfaTable()
 448         records = table.find({'type': 'user', 'hrn': newList})
 449         for rec in records:
 450             newIdList.append(rec['pointer'])
 451
 452         # build a list of the old person ids from the person_ids field
 453         if oldRecord:
 454             oldIdList = oldRecord.get("person_ids", [])
 455             containerId = oldRecord.get_pointer()
 456         else:
 457             # if oldRecord==None, then we are doing a Register, instead of an
 458             # update.
 459             oldIdList = []
 460             containerId = record.get_pointer()
 461
 462     # add people who are in the new list, but not the oldList
 463         for personId in newIdList:
 464             if not (personId in oldIdList):
 465                 addFunc(self.plauth, personId, containerId)
 466
 467         # remove people who are in the old list, but not the new list
 468         for personId in oldIdList:
 469             if not (personId in newIdList):
 470                 delFunc(self.plauth, personId, containerId)
 471
 472     def update_membership(self, oldRecord, record):
 473         if record.type == "slice":
 474             self.update_membership_list(oldRecord, record, 'researcher',
 475                                         self.plshell.AddPersonToSlice,
 476                                         self.plshell.DeletePersonFromSlice)
 477         elif record.type == "authority":
 478             # xxx TODO
 479             pass