X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=PLC%2FMethods%2FRefreshPeer.py;h=515bf9bc9bd3c8f128a7ec9f443b0beb2a7b8bfc;hb=4b98bfe145e937f5773cd77c3ffcbeb233911d74;hp=3657e3b9b9199e57018aca892496affdaf80e2c7;hpb=56b2bed39de364cd20ff76fb022f7f65a5f21410;p=plcapi.git diff --git a/PLC/Methods/RefreshPeer.py b/PLC/Methods/RefreshPeer.py index 3657e3b..515bf9b 100644 --- a/PLC/Methods/RefreshPeer.py +++ b/PLC/Methods/RefreshPeer.py @@ -2,71 +2,424 @@ # Thierry Parmentelat - INRIA # -import xmlrpclib +import time +from PLC.Debug import log from PLC.Faults import * from PLC.Method import Method from PLC.Parameter import Parameter, Mixed from PLC.Auth import Auth from PLC.Peers import Peer, Peers +from PLC.Sites import Site, Sites from PLC.Persons import Person, Persons -from PLC.ForeignNodes import ForeignNode, ForeignNodes - +from PLC.KeyTypes import KeyType, KeyTypes +from PLC.Keys import Key, Keys +from PLC.BootStates import BootState, BootStates +from PLC.Nodes import Node, Nodes +from PLC.SliceInstantiations import SliceInstantiations +from PLC.Slices import Slice, Slices class RefreshPeer(Method): """ - Query a peer PLC for its list of nodes, and refreshes - the local database accordingly - - Returns the number of new nodes from that peer - may be negative + Fetches node and slice data from the specified peer and caches it + locally; also deletes stale entries. Returns 1 if successful, + faults otherwise. """ - + roles = ['admin'] - - accepts = [ Auth(), - Parameter (int, "Peer id") ] - - returns = Parameter(int, "Delta in number of foreign nodes attached to that peer") - - def call (self, auth, peer_id): - - ### retrieve peer info - peers = Peers (self.api,[peer_id]) + + accepts = [ + Auth(), + Mixed(Peer.fields['peer_id'], + Peer.fields['peername']), + ] + + returns = Parameter(int, "1 if successful") + + def call(self, auth, peer_id_or_peername): + # Get peer + peers = Peers(self.api, [peer_id_or_peername]) if not peers: - raise PLCInvalidArgument,'no such peer_id:%d'%peer_id - peer=peers[0] - - ### retrieve account info - person_id = peer['person_id'] - persons = Persons (self.api,[person_id]) - if not persons: - raise PLCInvalidArgument,'no such person_id:%d'%person_id - person = persons[0] - - ### build up foreign auth - auth={ 'Username': person['email'], - 'AuthMethod' : 'password', - 'AuthString' : person['password'], - 'Role' : 'admin' } - - ## connect to the peer's API - url=peer['peer_url']+"/PLCAPI/" - print 'url=',url - apiserver = xmlrpclib.Server (url) - print 'auth=',auth - current_peer_nodes = apiserver.GetNodes(auth) - print 'current_peer_nodes',current_peer_nodes - - ## manual feed for tests - n11={'session': None, 'slice_ids': [], 'nodegroup_ids': [], 'last_updated': 1162884349, 'version': None, 'nodenetwork_ids': [], 'boot_state': 'inst', 'hostname': 'n11.plc1.org', 'site_id': 1, 'ports': None, 'pcu_ids': [], 'boot_nonce': None, 'node_id': 1, 'root_person_ids': [], 'key': None, 'date_created': 1162884349, 'model': None, 'conf_file_ids': [], 'ssh_rsa_key': None} - n12={'session': None, 'slice_ids': [], 'nodegroup_ids': [], 'last_updated': 1162884349, 'version': None, 'nodenetwork_ids': [], 'boot_state': 'inst', 'hostname': 'n12.plc1.org', 'site_id': 1, 'ports': None, 'pcu_ids': [], 'boot_nonce': None, 'node_id': 1, 'root_person_ids': [], 'key': None, 'date_created': 1162884349, 'model': None, 'conf_file_ids': [], 'ssh_rsa_key': None} - n21={'session': None, 'slice_ids': [], 'nodegroup_ids': [], 'last_updated': 1162884349, 'version': None, 'nodenetwork_ids': [], 'boot_state': 'boot', 'hostname': 'n21.plc2.org', 'site_id': 1, 'ports': None, 'pcu_ids': [], 'boot_nonce': None, 'node_id': 1, 'root_person_ids': [], 'key': None, 'date_created': 1162884349, 'model': None, 'conf_file_ids': [], 'ssh_rsa_key': None} - n22={'session': None, 'slice_ids': [], 'nodegroup_ids': [], 'last_updated': 1162884349, 'version': None, 'nodenetwork_ids': [], 'boot_state': 'boot', 'hostname': 'n22.plc2.org', 'site_id': 1, 'ports': None, 'pcu_ids': [], 'boot_nonce': None, 'node_id': 1, 'root_person_ids': [], 'key': None, 'date_created': 1162884349, 'model': None, 'conf_file_ids': [], 'ssh_rsa_key': None} - -# current_peer_nodes = [] -# print 'current_peer_nodes',current_peer_nodes - - nb_new_nodes = peer.refresh_nodes(current_peer_nodes) - - return nb_new_nodes + raise PLCInvalidArgument, "No such peer '%s'" % unicode(peer_id_or_peername) + peer = peers[0] + peer_id = peer['peer_id'] + + # Connect to peer API + peer.connect() + + timers = {} + + # Get peer data + start = time.time() + peer_tables = peer.GetPeerData() + timers['transport'] = time.time() - start - peer_tables['db_time'] + timers['peer_db'] = peer_tables['db_time'] + + def sync(objects, peer_objects, classobj): + """ + Synchronizes two dictionaries of objects. objects should + be a dictionary of local objects keyed on their foreign + identifiers. peer_objects should be a dictionary of + foreign objects keyed on their local (i.e., foreign to us) + identifiers. Returns a final dictionary of local objects + keyed on their foreign identifiers. + """ + + synced = {} + + # Delete stale objects + for peer_object_id, object in objects.iteritems(): + if peer_object_id not in peer_objects: + object.delete(commit = False) + print classobj, "object %d deleted" % object[object.primary_key] + + # Add/update new/existing objects + for peer_object_id, peer_object in peer_objects.iteritems(): + if peer_object_id in objects: + # Update existing object + object = objects[peer_object_id] + + # Replace foreign identifier with existing local + # identifier temporarily for the purposes of + # comparison. + peer_object[object.primary_key] = object[object.primary_key] + + # Must use __eq__() instead of == since + # peer_object may be a raw dict instead of a Peer + # object. + if not object.__eq__(peer_object): + # Only update intrinsic fields + object.update(object.db_fields(peer_object)) + sync = True + dbg = "changed" + else: + sync = False + dbg = None + + # Restore foreign identifier + peer_object[object.primary_key] = peer_object_id + else: + # Add new object + object = classobj(self.api, peer_object) + # Replace foreign identifier with new local identifier + del object[object.primary_key] + sync = True + dbg = "added" + + if sync: + try: + object.sync(commit = False) + except PLCInvalidArgument, err: + # Skip if validation fails + # XXX Log an event instead of printing to logfile + print >> log, "Warning: Skipping invalid", \ + peer['peername'], object.__class__.__name__, \ + ":", peer_object, ":", err + continue + + synced[peer_object_id] = object + + if dbg: + print >> log, peer['peername'], classobj(self.api).__class__.__name__, object[object.primary_key], dbg + + return synced + + # + # Synchronize foreign sites + # + + start = time.time() + + # Compare only the columns returned by the GetPeerData() call + if peer_tables['Sites']: + columns = peer_tables['Sites'][0].keys() + else: + columns = None + + # Keyed on foreign site_id + old_peer_sites = Sites(self.api, {'peer_id': peer_id}, columns).dict('peer_site_id') + sites_at_peer = dict([(site['site_id'], site) for site in peer_tables['Sites']]) + + # Synchronize new set (still keyed on foreign site_id) + peer_sites = sync(old_peer_sites, sites_at_peer, Site) + + for peer_site_id, site in peer_sites.iteritems(): + # Bind any newly cached sites to peer + if peer_site_id not in old_peer_sites: + peer.add_site(site, peer_site_id, commit = False) + site['peer_id'] = peer_id + site['peer_site_id'] = peer_site_id + + timers['site'] = time.time() - start + + # + # XXX Synchronize foreign key types + # + + key_types = KeyTypes(self.api).dict() + + # + # Synchronize foreign keys + # + + start = time.time() + + # Compare only the columns returned by the GetPeerData() call + if peer_tables['Keys']: + columns = peer_tables['Keys'][0].keys() + else: + columns = None + + # Keyed on foreign key_id + old_peer_keys = Keys(self.api, {'peer_id': peer_id}, columns).dict('peer_key_id') + keys_at_peer = dict([(key['key_id'], key) for key in peer_tables['Keys']]) + + # Fix up key_type references + for peer_key_id, key in keys_at_peer.items(): + if key['key_type'] not in key_types: + # XXX Log an event instead of printing to logfile + print >> log, "Warning: Skipping invalid %s key:" % peer['peername'], \ + key, ": invalid key type", key['key_type'] + del keys_at_peer[peer_key_id] + continue + + # Synchronize new set (still keyed on foreign key_id) + peer_keys = sync(old_peer_keys, keys_at_peer, Key) + for peer_key_id, key in peer_keys.iteritems(): + # Bind any newly cached keys to peer + if peer_key_id not in old_peer_keys: + peer.add_key(key, peer_key_id, commit = False) + key['peer_id'] = peer_id + key['peer_key_id'] = peer_key_id + + timers['keys'] = time.time() - start + + # + # Synchronize foreign users + # + + start = time.time() + + # Compare only the columns returned by the GetPeerData() call + if peer_tables['Persons']: + columns = peer_tables['Persons'][0].keys() + else: + columns = None + + # Keyed on foreign person_id + old_peer_persons = Persons(self.api, {'peer_id': peer_id}, columns).dict('peer_person_id') + persons_at_peer = dict([(peer_person['person_id'], peer_person) \ + for peer_person in peer_tables['Persons']]) + + # XXX Do we care about membership in foreign site(s)? + + # Synchronize new set (still keyed on foreign person_id) + peer_persons = sync(old_peer_persons, persons_at_peer, Person) + + # transcoder : retrieve a local key_id from a peer_key_id + key_transcoder = dict ( [ (key['key_id'],key['peer_key_id']) for key in peer_keys.values()]) + + for peer_person_id, person in peer_persons.iteritems(): + # Bind any newly cached users to peer + if peer_person_id not in old_peer_persons: + peer.add_person(person, peer_person_id, commit = False) + person['peer_id'] = peer_id + person['peer_person_id'] = peer_person_id + person['key_ids'] = [] + + # User as viewed by peer + peer_person = persons_at_peer[peer_person_id] + + # Foreign keys currently belonging to the user + old_person_key_ids = [key_transcoder[key_id] for key_id in person['key_ids']] + + # Foreign keys that should belong to the user + # this is basically peer_person['key_ids'], we just check it makes sense + # (e.g. we might have failed importing it) + person_key_ids = [ x for x in peer_person['key_ids'] if x in peer_keys] + + # Remove stale keys from user + for peer_key_id in (set(old_person_key_ids) - set(person_key_ids)): + person.remove_key(peer_keys[peer_key_id], commit = False) + + # Add new keys to user + for peer_key_id in (set(person_key_ids) - set(old_person_key_ids)): + person.add_key(peer_keys[peer_key_id], commit = False) + + timers['persons'] = time.time() - start + + # + # XXX Synchronize foreign boot states + # + + boot_states = BootStates(self.api).dict() + + # + # Synchronize foreign nodes + # + + start = time.time() + + # Compare only the columns returned by the GetPeerData() call + if peer_tables['Nodes']: + columns = peer_tables['Nodes'][0].keys() + else: + columns = None + + # Keyed on foreign node_id + old_peer_nodes = Nodes(self.api, {'peer_id': peer_id}, columns).dict('peer_node_id') + nodes_at_peer = dict([(node['node_id'], node) \ + for node in peer_tables['Nodes']]) + + # Fix up site_id and boot_states references + for peer_node_id, node in nodes_at_peer.items(): + errors = [] + if node['site_id'] not in peer_sites: + errors.append("invalid site %d" % node['site_id']) + if node['boot_state'] not in boot_states: + errors.append("invalid boot state %s" % node['boot_state']) + if errors: + # XXX Log an event instead of printing to logfile + print >> log, "Warning: Skipping invalid %s node:" % peer['peername'], \ + node, ":", ", ".join(errors) + del nodes_at_peer[peer_node_id] + continue + else: + node['site_id'] = peer_sites[node['site_id']]['site_id'] + + # Synchronize new set + peer_nodes = sync(old_peer_nodes, nodes_at_peer, Node) + + for peer_node_id, node in peer_nodes.iteritems(): + # Bind any newly cached foreign nodes to peer + if peer_node_id not in old_peer_nodes: + peer.add_node(node, peer_node_id, commit = False) + node['peer_id'] = peer_id + node['peer_node_id'] = peer_node_id + + timers['nodes'] = time.time() - start + + # + # Synchronize local nodes + # + + start = time.time() + + # Keyed on local node_id + local_nodes = Nodes(self.api).dict() + + for node in peer_tables['PeerNodes']: + # Foreign identifier for our node as maintained by peer + peer_node_id = node['node_id'] + # Local identifier for our node as cached by peer + node_id = node['peer_node_id'] + if node_id in local_nodes: + # Still a valid local node, add it to the synchronized + # set of local node objects keyed on foreign node_id. + peer_nodes[peer_node_id] = local_nodes[node_id] + + timers['local_nodes'] = time.time() - start + + # + # XXX Synchronize foreign slice instantiation states + # + + slice_instantiations = SliceInstantiations(self.api).dict() + + # + # Synchronize foreign slices + # + + start = time.time() + + # Compare only the columns returned by the GetPeerData() call + if peer_tables['Slices']: + columns = peer_tables['Slices'][0].keys() + else: + columns = None + + # Keyed on foreign slice_id + old_peer_slices = Slices(self.api, {'peer_id': peer_id}, columns).dict('peer_slice_id') + slices_at_peer = dict([(slice['slice_id'], slice) \ + for slice in peer_tables['Slices']]) + + # Fix up site_id, instantiation, and creator_person_id references + for peer_slice_id, slice in slices_at_peer.items(): + errors = [] + if slice['site_id'] not in peer_sites: + errors.append("invalid site %d" % slice['site_id']) + if slice['instantiation'] not in slice_instantiations: + errors.append("invalid instantiation %s" % slice['instantiation']) + if slice['creator_person_id'] not in peer_persons: + # Just NULL it out + slice['creator_person_id'] = None + else: + slice['creator_person_id'] = peer_persons[slice['creator_person_id']]['person_id'] + if errors: + print >> log, "Warning: Skipping invalid %s slice:" % peer['peername'], \ + slice, ":", ", ".join(errors) + del slices_at_peer[peer_slice_id] + continue + else: + slice['site_id'] = peer_sites[slice['site_id']]['site_id'] + + # Synchronize new set + peer_slices = sync(old_peer_slices, slices_at_peer, Slice) + + # transcoder : retrieve a local node_id from a peer_node_id + node_transcoder = dict ( [ (node['node_id'],node['peer_node_id']) for node in peer_nodes.values()]) + person_transcoder = dict ( [ (person['person_id'],person['peer_person_id']) for person in peer_persons.values()]) + + for peer_slice_id, slice in peer_slices.iteritems(): + # Bind any newly cached foreign slices to peer + if peer_slice_id not in old_peer_slices: + peer.add_slice(slice, peer_slice_id, commit = False) + slice['peer_id'] = peer_id + slice['peer_slice_id'] = peer_slice_id + slice['node_ids'] = [] + slice['person_ids'] = [] + + # Slice as viewed by peer + peer_slice = slices_at_peer[peer_slice_id] + + # Nodes that are currently part of the slice + old_slice_node_ids = [ node_transcoder[node_id] for node_id in slice['node_ids']] + + # Nodes that should be part of the slice + slice_node_ids = [ x for x in peer_slice['node_ids'] if x in peer_nodes] + + # Remove stale nodes from slice + for node_id in (set(old_slice_node_ids) - set(slice_node_ids)): + slice.remove_node(peer_nodes[node_id], commit = False) + + # Add new nodes to slice + for node_id in (set(slice_node_ids) - set(old_slice_node_ids)): + slice.add_node(peer_nodes[node_id], commit = False) + + # N.B.: Local nodes that may have been added to the slice + # by hand, are removed. In other words, don't do this. + + # Foreign users that are currently part of the slice + old_slice_person_ids = [ person_transcoder[person_id] for person_id in slice['person_ids']] + + # Foreign users that should be part of the slice + slice_person_ids = [ x for x in peer_slice['person_ids'] if x in peer_persons] + + # Remove stale users from slice + for peer_person_id in (set(old_slice_person_ids) - set(slice_person_ids)): + slice.remove_person(peer_persons[peer_person_id], commit = False) + + # Add new users to slice + for peer_person_id in (set(slice_person_ids) - set(old_slice_person_ids)): + slice.add_person(peer_persons[peer_person_id], commit = False) + + # N.B.: Local users that may have been added to the slice + # by hand, are not touched. + + timers['slices'] = time.time() - start + + # Update peer itself and commit + peer.sync(commit = True) + + return timers