sfa/plc/api-dev.py

   1 #
   2 # SFA XML-RPC and SOAP interfaces
   3 #
   4 ### $Id: api.py 17793 2010-04-26 21:40:57Z tmack $
   5 ### $URL: https://svn.planet-lab.org/svn/sfa/trunk/sfa/plc/api.py $
   6 #
   7
   8 import sys
   9 import os
  10 import traceback
  11 import string
  12 import xmlrpclib
  13
  14 import sfa.util.sfalogging
  15 from sfa.trust.auth import Auth
  16 from sfa.util.config import *
  17 from sfa.util.faults import *
  18 from sfa.util.debug import *
  19 from sfa.trust.rights import *
  20 from sfa.trust.credential import *
  21 from sfa.trust.certificate import *
  22 from sfa.util.namespace import *
  23 from sfa.util.api import *
  24 from sfa.util.nodemanager import NodeManager
  25 from collections import defaultdict
  26
  27
  28 class RecordInfo():
  29
  30     shell = None
  31     auth = None
  32
  33     # pl records
  34     sites = {}
  35     slices = {}
  36     persons = {}
  37     nodes = {}
  38     keys = {}
  39
  40     # sfa records
  41     sfa_authorities = {}
  42     sfa_slices = {}
  43     sfa_users = {}
  44     sfa_nodes = {}
  45
  46     records = []
  47
  48     def __init__(self, api, records):
  49         self.api = api
  50         self.shell = api.plshell
  51         self.auth = api.plauth
  52
  53         site_ids = []
  54         slice_ids = []
  55         person_ids = []
  56         node_ids = []
  57
  58         # put records into groups based on types
  59         for record in records:
  60             pointer = record['pointer']
  61             if record['type'] == 'authority':
  62                 self.sfa_authorities[pointer] = record
  63                 self.records.append(record)
  64                 site_ids.append(record['pointer'])
  65             elif record['type'] == 'slice':
  66                 self.sfa_slices[pointer] = record
  67                 self.records.append(record)
  68                 slice_ids.append(record['pointer'])
  69             elif record['type'] == 'user':
  70                 self.sfa_users[pointer] = record
  71                 self.records.append(record)
  72                 person_ids.append(record['pointer'])
  73             elif record['type'] == 'node':
  74                 self.sfa_nodes[pointer] = record
  75                 self.records.append(record)
  76                 node_ids.append(record['pointer'])
  77
  78         # get pl info for these records
  79         self.update_pl_sites(site_ids)
  80         self.update_pl_slices(slice_ids)
  81         self.update_pl_persons(person_ids)
  82         self.update_pl_nodes(node_ids)
  83
  84         site_ids = []
  85         slice_ids = []
  86         person_ids = []
  87         node_ids = []
  88         # now get pl records for all ids associated with
  89         # these records
  90         for record in records:
  91             if 'site_id' in record:
  92                 site_ids.append(record['site_id'])
  93             if 'site_ids' in records:
  94                 site_ids.extend(record['site_ids'])
  95             if 'person_ids' in record:
  96                 person_ids.extend(record['person_ids'])
  97             if 'slice_ids' in record:
  98                 slice_ids.extend(record['slice_ids'])
  99             if 'node_ids' in record:
 100                 node_ids.extend(record['node_ids'])
 101
 102         # get pl info for these records
 103         self.update_pl_sites(site_ids)
 104         self.update_pl_slices(slice_ids)
 105         self.update_pl_persons(person_ids)
 106         self.update_pl_nodes(node_ids)
 107
 108         # convert pl ids to hrns
 109         self.update_hrns()
 110
 111         # update sfa info
 112         self.update_sfa_info(person_ids)
 113
 114     def update_pl_sites(self, site_ids):
 115         """
 116         Update site records with PL info
 117         """
 118         if not site_ids:
 119             return
 120         sites = self.shell.GetSites(self.auth, site_ids)
 121         for site in sites:
 122             site_id = site['site_id']
 123             self.sites[site_id] = site
 124             if site_id in self.sfa_authorities:
 125                 self.sfa_authorities[site_id].update(site)
 126
 127     def update_pl_slices(self, slice_ids):
 128         """
 129         Update slice records with PL info
 130         """
 131         if not slice_ids:
 132             return
 133         slices = self.shell.GetSlices(self.auth, slice_ids)
 134         for slice in slices:
 135             slice_id = slice['slice_id']
 136             self.slices[slice_id] = slice
 137             if slice_id in self.sfa_slices:
 138                 self.sfa_slices[slice_id].update(slice)
 139
 140     def update_pl_persons(self, person_ids):
 141         """
 142         Update person records with PL info
 143         """
 144         key_ids = []
 145         if not person_ids:
 146             return
 147         persons = self.shell.GetPersons(self.auth, person_ids)
 148         for person in persons:
 149             person_id = person['person_id']
 150             self.persons[person_id] = person
 151             key_ids.extend(person['key_ids'])
 152             if person_id in self.sfa_users:
 153                 self.sfa_users[person_id].update(person)
 154         self.update_pl_keys(key_ids)
 155
 156     def update_pl_keys(self, key_ids):
 157         """
 158         Update user records with PL public key info
 159         """
 160         if not key_ids:
 161             return
 162         keys = self.shell.GetKeys(self.auth, key_ids)
 163         for key in keys:
 164             person_id = key['person_id']
 165             self.keys[key['key_id']] = key
 166             if person_id in self.sfa_users:
 167                 person = self.sfa_users[person_id]
 168                 if not 'keys' in person:
 169                     person['keys'] = [key['key']]
 170                 else:
 171                     person['keys'].append(key['key'])
 172
 173     def update_pl_nodes(self, node_ids):
 174         """
 175         Update node records with PL info
 176         """
 177         if not node_ids:
 178             return
 179         nodes = self.shell.GetNodes(self.auth, node_ids)
 180         for node in nodes:
 181             node_id = node['node_id']
 182             self.nodes[node['node_id']] = node
 183             if node_id in self.sfa_nodes:
 184                 self.sfa_nodes[node_id].update(node)
 185
 186
 187     def update_hrns(self):
 188         """
 189         Convert pl ids to hrns
 190         """
 191         for record in self.records:
 192             # get all necessary data
 193             type = record['type']
 194             pointer = record['pointer']
 195             auth_hrn = self.api.hrn
 196             login_base = ''
 197             if pointer == -1:
 198                 continue
 199
 200             if 'site_id' in record:
 201                 site = self.sites[record['site_id']]
 202                 login_base = site['login_base']
 203                 record['site'] = ".".join([auth_hrn, login_base])
 204             if 'person_ids' in record:
 205                 emails = [self.persons[person_id]['email'] for person_id in record['person_ids'] \
 206                           if person_id in self.persons]
 207                 usernames = [email.split('@')[0] for email in emails]
 208                 person_hrns = [".".join([auth_hrn, login_base, username]) for username in usernames]
 209                 record['persons'] = person_hrns
 210             if 'slice_ids' in record:
 211                 slicenames = [self.slices[slice_id]['name'] for slice_id in record['slice_ids'] \
 212                               if slice_id in self.slices]
 213                 slice_hrns = [slicename_to_hrn(auth_hrn, slicename) for slicename in slicenames]
 214                 record['slices'] = slice_hrns
 215             if 'node_ids' in record:
 216                 hostnames = [self.nodes[node_id]['hostname'] for node_id in record['node_ids'] \
 217                              if node_id in self.nodes]
 218                 node_hrns = [hostname_to_hrn(auth_hrn, login_base, hostname) for hostname in hostnames]
 219                 record['nodes'] = node_hrns
 220             if 'site_ids' in record:
 221                 login_bases = [self.sites[site_id]['login_base'] for site_id in record['site_ids'] \
 222                                if site_id in self.sites]
 223                 site_hrns = [".".join([auth_hrn, lbase]) for lbase in login_bases]
 224                 record['sites'] = site_hrns
 225
 226     def update_sfa_info(self, person_ids):
 227         from sfa.util.table import SfaTable
 228         table = SfaTable()
 229         persons = table.find({'type': 'user', 'pointer': person_ids})
 230         # create a hrns keyed on the sfa record's pointer.
 231         # Its possible for  multiple records to have the same pointer so
 232         # the dict's value will be a list of hrns.
 233         person_dict = defaultdict(list)
 234         for person in persons:
 235             person_dict[person['pointer']].append(person['hrn'])
 236
 237         def startswith(prefix, values):
 238             return [value for value in values if value.startswith(prefix)]
 239
 240         for record in self.records:
 241             authority = record['authority']
 242             if record['pointer'] == -1:
 243                 continue
 244
 245             if record['type'] == 'slice':
 246                 # all slice users are researchers
 247                 record['PI'] = []
 248                 record['researchers'] = []
 249                 for person_id in record['person_ids']:
 250                     record['researchers'].extend(person_dict[person_id])
 251                 # also add the pis at the slice's site
 252                 site = self.sites[record['site_id']]
 253                 for person_id in site['person_ids']:
 254                     person = self.persons[person_id]
 255                     if 'pi' in person['roles']:
 256                         # PLCAPI doesn't support per site roles
 257                         # (a pi has the pi role at every site he belongs to).
 258                         # We shouldnt allow this in SFA
 259                         record['PI'].extend(startswith(authority, person_dict[person_id]))
 260
 261             elif record['type'] == 'authority':
 262                 record['PI'] = []
 263                 record['operator'] = []
 264                 record['owner'] = []
 265                 for person_id in record['person_ids']:
 266                     person = self.persons[person_id]
 267                     if 'pi' in person['roles']:
 268                         # only get PI's at this site
 269                         record['PI'].extend(startswith(record['hrn'], person_dict[person_id]))
 270                     if 'tech' in person['roles']:
 271                         # only get PI's at this site
 272                         record['operator'].extend(startswith(record['hrn'], person_dict[person_id]))
 273                     if 'admin' in person['roles']:
 274                         record['owner'].extend(startswith(record['hrn'], person_dict[person_id]))
 275
 276             elif record['type'] == 'node':
 277                 record['dns'] = record['hostname']
 278
 279             elif record['type'] == 'user':
 280                 record['email'] = record['email']
 281
 282
 283
 284
 285
 286     def get_records(self):
 287         return self.records
 288
 289
 290 class SfaAPI(BaseAPI):
 291
 292     # flat list of method names
 293     import sfa.methods
 294     methods = sfa.methods.all
 295
 296     def __init__(self, config = "/etc/sfa/sfa_config.py", encoding = "utf-8",
 297                  methods='sfa.methods', peer_cert = None, interface = None,
 298                 key_file = None, cert_file = None, cache = None):
 299         BaseAPI.__init__(self, config=config, encoding=encoding, methods=methods, \
 300                          peer_cert=peer_cert, interface=interface, key_file=key_file, \
 301                          cert_file=cert_file, cache=cache)
 302
 303         self.encoding = encoding
 304
 305         from sfa.util.table import SfaTable
 306         self.SfaTable = SfaTable
 307         # Better just be documenting the API
 308         if config is None:
 309             return
 310
 311         # Load configuration
 312         self.config = Config(config)
 313         self.auth = Auth(peer_cert)
 314         self.interface = interface
 315         self.key_file = key_file
 316         self.key = Keypair(filename=self.key_file)
 317         self.cert_file = cert_file
 318         self.cert = Certificate(filename=self.cert_file)
 319         self.credential = None
 320         # Initialize the PLC shell only if SFA wraps a myPLC
 321         rspec_type = self.config.get_aggregate_type()
 322         if (rspec_type == 'pl' or rspec_type == 'vini'):
 323             self.plshell = self.getPLCShell()
 324             self.plshell_version = "4.3"
 325
 326         self.hrn = self.config.SFA_INTERFACE_HRN
 327         self.time_format = "%Y-%m-%d %H:%M:%S"
 328         self.logger=sfa.util.sfalogging.logger
 329
 330     def getPLCShell(self):
 331         self.plauth = {'Username': self.config.SFA_PLC_USER,
 332                        'AuthMethod': 'password',
 333                        'AuthString': self.config.SFA_PLC_PASSWORD}
 334
 335         self.plshell_type = 'xmlrpc'
 336         # connect via xmlrpc
 337         url = self.config.SFA_PLC_URL
 338         shell = xmlrpclib.Server(url, verbose = 0, allow_none = True)
 339         return shell
 340
 341     def getCredential(self):
 342         if self.interface in ['registry']:
 343             return self.getCredentialFromLocalRegistry()
 344         else:
 345             return self.getCredentialFromRegistry()
 346
 347     def getCredentialFromRegistry(self):
 348         """
 349         Get our credential from a remote registry
 350         """
 351         type = 'authority'
 352         path = self.config.SFA_DATA_DIR
 353         filename = ".".join([self.interface, self.hrn, type, "cred"])
 354         cred_filename = path + os.sep + filename
 355         try:
 356             credential = Credential(filename = cred_filename)
 357             return credential.save_to_string(save_parents=True)
 358         except IOError:
 359             from sfa.server.registry import Registries
 360             registries = Registries(self)
 361             registry = registries[self.hrn]
 362             cert_string=self.cert.save_to_string(save_parents=True)
 363             # get self credential
 364             self_cred = registry.get_self_credential(cert_string, type, self.hrn)
 365             # get credential
 366             cred = registry.get_credential(self_cred, type, self.hrn)
 367
 368             # save cred to file
 369             Credential(string=cred).save_to_file(cred_filename, save_parents=True)
 370             return cred
 371
 372     def getCredentialFromLocalRegistry(self):
 373         """
 374         Get our current credential directly from the local registry.
 375         """
 376
 377         hrn = self.hrn
 378         auth_hrn = self.auth.get_authority(hrn)
 379
 380         # is this a root or sub authority
 381         if not auth_hrn or hrn == self.config.SFA_INTERFACE_HRN:
 382             auth_hrn = hrn
 383         auth_info = self.auth.get_auth_info(auth_hrn)
 384         table = self.SfaTable()
 385         records = table.findObjects(hrn)
 386         if not records:
 387             raise RecordNotFound
 388         record = records[0]
 389         type = record['type']
 390         object_gid = record.get_gid_object()
 391         new_cred = Credential(subject = object_gid.get_subject())
 392         new_cred.set_gid_caller(object_gid)
 393         new_cred.set_gid_object(object_gid)
 394         new_cred.set_issuer(key=auth_info.get_pkey_object(), subject=auth_hrn)
 395         new_cred.set_pubkey(object_gid.get_pubkey())
 396         r1 = determine_rights(type, hrn)
 397         new_cred.set_privileges(r1)
 398
 399         auth_kind = "authority,ma,sa"
 400
 401         new_cred.set_parent(self.auth.hierarchy.get_auth_cred(auth_hrn, kind=auth_kind))
 402
 403         new_cred.encode()
 404         new_cred.sign()
 405
 406         return new_cred.save_to_string(save_parents=True)
 407
 408
 409     def loadCredential (self):
 410         """
 411         Attempt to load credential from file if it exists. If it doesnt get
 412         credential from registry.
 413         """
 414
 415         # see if this file exists
 416         # XX This is really the aggregate's credential. Using this is easier than getting
 417         # the registry's credential from iteslf (ssl errors).
 418         ma_cred_filename = self.config.SFA_DATA_DIR + os.sep + self.interface + self.hrn + ".ma.cred"
 419         try:
 420             self.credential = Credential(filename = ma_cred_filename)
 421         except IOError:
 422             self.credential = self.getCredentialFromRegistry()
 423
 424     ##
 425     # Convert SFA fields to PLC fields for use when registering up updating
 426     # registry record in the PLC database
 427     #
 428     # @param type type of record (user, slice, ...)
 429     # @param hrn human readable name
 430     # @param sfa_fields dictionary of SFA fields
 431     # @param pl_fields dictionary of PLC fields (output)
 432
 433     def sfa_fields_to_pl_fields(self, type, hrn, record):
 434
 435         def convert_ints(tmpdict, int_fields):
 436             for field in int_fields:
 437                 if field in tmpdict:
 438                     tmpdict[field] = int(tmpdict[field])
 439
 440         pl_record = {}
 441         #for field in record:
 442         #    pl_record[field] = record[field]
 443
 444         if type == "slice":
 445             if not "instantiation" in pl_record:
 446                 pl_record["instantiation"] = "plc-instantiated"
 447             pl_record["name"] = hrn_to_pl_slicename(hrn)
 448             if "url" in record:
 449                pl_record["url"] = record["url"]
 450             if "description" in record:
 451                 pl_record["description"] = record["description"]
 452             if "expires" in record:
 453                 pl_record["expires"] = int(record["expires"])
 454
 455         elif type == "node":
 456             if not "hostname" in pl_record:
 457                 if not "hostname" in record:
 458                     raise MissingSfaInfo("hostname")
 459                 pl_record["hostname"] = record["hostname"]
 460             if not "model" in pl_record:
 461                 pl_record["model"] = "geni"
 462
 463         elif type == "authority":
 464             pl_record["login_base"] = hrn_to_pl_login_base(hrn)
 465
 466             if not "name" in pl_record:
 467                 pl_record["name"] = hrn
 468
 469             if not "abbreviated_name" in pl_record:
 470                 pl_record["abbreviated_name"] = hrn
 471
 472             if not "enabled" in pl_record:
 473                 pl_record["enabled"] = True
 474
 475             if not "is_public" in pl_record:
 476                 pl_record["is_public"] = True
 477
 478         return pl_record
 479
 480
 481     def fill_record_info(self, records):
 482         """
 483         Given a SFA record, fill in the PLC specific and SFA specific
 484         fields in the record.
 485         """
 486         if not isinstance(records, list):
 487             records = [records]
 488
 489         record_info = RecordInfo(self, records)
 490         return record_info.get_records()
 491
 492     def update_membership_list(self, oldRecord, record, listName, addFunc, delFunc):
 493         # get a list of the HRNs tht are members of the old and new records
 494         if oldRecord:
 495             oldList = oldRecord.get(listName, [])
 496         else:
 497             oldList = []
 498         newList = record.get(listName, [])
 499
 500         # if the lists are the same, then we don't have to update anything
 501         if (oldList == newList):
 502             return
 503
 504         # build a list of the new person ids, by looking up each person to get
 505         # their pointer
 506         newIdList = []
 507         table = self.SfaTable()
 508         records = table.find({'type': 'user', 'hrn': newList})
 509         for rec in records:
 510             newIdList.append(rec['pointer'])
 511
 512         # build a list of the old person ids from the person_ids field
 513         if oldRecord:
 514             oldIdList = oldRecord.get("person_ids", [])
 515             containerId = oldRecord.get_pointer()
 516         else:
 517             # if oldRecord==None, then we are doing a Register, instead of an
 518             # update.
 519             oldIdList = []
 520             containerId = record.get_pointer()
 521
 522     # add people who are in the new list, but not the oldList
 523         for personId in newIdList:
 524             if not (personId in oldIdList):
 525                 addFunc(self.plauth, personId, containerId)
 526
 527         # remove people who are in the old list, but not the new list
 528         for personId in oldIdList:
 529             if not (personId in newIdList):
 530                 delFunc(self.plauth, personId, containerId)
 531
 532     def update_membership(self, oldRecord, record):
 533         if record.type == "slice":
 534             self.update_membership_list(oldRecord, record, 'researcher',
 535                                         self.plshell.AddPersonToSlice,
 536                                         self.plshell.DeletePersonFromSlice)
 537         elif record.type == "authority":
 538             # xxx TODO
 539             pass
 540
 541
 542
 543 class ComponentAPI(BaseAPI):
 544
 545     def __init__(self, config = "/etc/sfa/sfa_config.py", encoding = "utf-8", methods='sfa.methods',
 546                  peer_cert = None, interface = None, key_file = None, cert_file = None):
 547
 548         BaseAPI.__init__(self, config=config, encoding=encoding, methods=methods, peer_cert=peer_cert,
 549                          interface=interface, key_file=key_file, cert_file=cert_file)
 550         self.encoding = encoding
 551
 552         # Better just be documenting the API
 553         if config is None:
 554             return
 555
 556         self.nodemanager = NodeManager(self.config)
 557
 558     def sliver_exists(self):
 559         sliver_dict = self.nodemanager.GetXIDs()
 560         if slicename in sliver_dict.keys():
 561             return True
 562         else:
 563             return False