class Cache:
- # an attempt to provide genericity in the caching algorithm
+ """
+ This class is the core of the RefreshPeer method's implementation,
+ that basically calls Cache:refresh_peer
+
+ It manages caching on a table-by-table basis, and performs required
+ transcoding from remote ids to local ids - that's the job of the
+ Transcoder class
+
+ For the tables (classes) that it handles, it uses the following
+ attributes
+ (*) primary_key is the name of the field where db indexes are stored
+ (*) class_key is used to implement equality on objects
+ (*) foreign_fields is the list of fields that are copied verbatim from
+ foreign objects
+ (*) foreign_xrefs is the list of columns that are subject to transcoding
+ (.) when obj[field] is an int, the transcoded value is directly
+ inserted in the table
+ (.) if it's a list, it is assumed that the relationship is maintained
+ in an external 2-column table. That's where the XrefTable class comes in
+
+ The relationship between slices, slice attribute types and slice attributes being
+ more complex, it is handled in a specifically customized version of update_table
+
+ Of course the order in which the tables are managed is important, with different
+ orders several iterations might be needed for the update process to converge.
+
+ Of course the timers field was introduced for optimization and could safely be removed
+ """
def __init__ (self, api, peer_id, peer_server, auth):
sql = "INSERT INTO %s (%s_id,%s_id) VALUES (%d,%d)"% \
(self.tablename,self.lowerclass1,self.lowerclass2,id1,id2)
-# below is Tony's code but it's badly broken. I'm not sure we care in fact.
+# below is Tony's code but it's badly broken. I'm not sure we care much in fact.
# if id2_set:
# sql = "INSERT INTO %s select %d, %d " % \
# self.tablename, id1, id2[0]
# peer_object_list list of objects at a given peer - e.g. peer.GetSlices()
# alien_xref_objs_dict : a dict {'classname':alien_obj_list} e.g. {'Node':peer.GetNodes()}
# we need an entry for each class mentioned in the class's foreign_xrefs
- # lambda_ignore : the alien objects are ignored if this returns true
def update_table (self,
classname,
alien_object_list,
alien_xref_objs_dict = {},
- lambda_ignore=lambda x:False,
report_name_conflicts = True):
verbose ("============================== entering update_table on",classname)
else:
local_object.uptodate=True
+ for alien_object in alien_object_list:
+ verbose ('+++ Got alien object',alien_object)
+
# scan the peer's local objects
for alien_object in alien_object_list:
object_name = alien_object[class_key]
- ### ignore, e.g. system-wide slices
- if lambda_ignore(alien_object):
- verbose('Ignoring',object_name)
- continue
+ verbose ('----- update_table (%s) - considering'%classname,object_name)
- verbose ('update_table (%s) - Considering'%classname,object_name)
-
# optimizing : avoid doing useless syncs
needs_sync = False
from PLC.SliceAttributeTypes import SliceAttributeTypes
from PLC.SliceAttributes import SliceAttribute, SliceAttributes
+ verbose ("============================== entering update_slice_attributes")
+
# init
peer_id = self.peer_id
timers={}
t_start=time.time()
# xxx see also GetPeerData - peer_id arg unused yet
- all_data = self.peer_server.GetPeerData (self.auth,0)
+ all_data = self.peer_server.GetPeerData (self.auth,self.api.config.PLC_NAME)
+
+ verbose ('Passed GetPeerData the name',self.api.config.PLC_NAME)
+ sks=all_data.keys()
+ sks.sort()
+ for k in sks:
+ f=all_data[k]
+ try:
+ verbose ('GetPeerData[%s] -> %d'%(k,len(f)))
+ except:
+ pass
t_acquired = time.time()
# refresh sites
plocal_slices = all_data['Slices-local']
all_slices = plocal_slices + all_data['Slices-peer']
- def is_system_slice (slice):
- return slice['creator_person_id'] == 1
+ # forget about ignoring remote system slices
+ # just update them too, we'll be able to filter them out later in GetSlivers
nb_new_slices = self.update_table ('Slice', plocal_slices,
{'Node': all_nodes,
'Person': all_persons,
- 'Site': all_sites},
- is_system_slice)
+ 'Site': all_sites})
t=time.time()
timers['process-slices']=t-t0
timers['time_all'] = t_end-t_start
### returned as-is by RefreshPeer
- return {'plcname':self.api.config.PLC_NAME,
+ return {
'new_sites':nb_new_sites,
'new_keys':nb_new_keys,
'new_nodes':nb_new_nodes,
from PLC.Peers import Peer, Peers
-can_update = lambda(k,v): k in ['peername','peer_url','person_id']
+can_update = lambda(k,v): k in ['peername','peer_url','auth_person_id']
class AddPeer (Method):
"""
Creates a peer entry in the database and returns its id
- Temporarily, requires to provide a person_id
+ Temporarily, requires to provide an auth_person_id
this is used to store the credentials that we'll
use when connecting to the peer's API
"""
raise PLCInvalidArgument, "No such node"
node = nodes[0]
- ### xxx here xxx
PLCCheckLocalNode(node,"DeleteNode")
# If we are not an admin, make sure that the caller is a
roles = ['admin']
accepts = [Auth(),
- Parameter (int, "Peer id"),
+ Mixed (Parameter (Peer.fields['peer_id']),
+ Parameter (Peer.fields['peername'])),
]
# for RefreshPeer
- returns = Parameter (dict,"Sites, Keys, Nodes, Persons, Slices")
+ returns = Parameter (dict,
+ "Sites-local Sites-peer Keys-local Keys-peer "
+ "Nodes-local Nodes-peer Persons-local Persons-peer "
+ "SliceAttibuteTypes-local SliceAttibuteTypes-peer "
+ "Slices-local Slices-peer "
+ "SliceAttributes-local SliceAttributes-peer")
- def call (self, auth, peer_id):
- # xxx a peer cannot yet compute it's peer_id under another plc
- # so we return all foreign objects by now
+ def call (self, auth, peer_id_or_peername):
+
+ # checking the argument
+ try:
+ peer_id = Peers(self.api,[peer_id_or_peername])[0]['peer_id']
+ except:
+ raise PLCInvalidArgument,'GetPeerData: no such peer %r'%peer_id_or_peername
t_start = time.time()
result = {
'Sites-local' : Sites (self.api,{'peer_id':None}),
- 'Sites-peer' : Sites (self.api,{'~peer_id':None}),
+ 'Sites-peer' : Sites (self.api,{'peer_id':peer_id}),
'Keys-local' : Keys (self.api,{'peer_id':None}),
- 'Keys-peer' : Keys (self.api,{'~peer_id':None}),
+ 'Keys-peer' : Keys (self.api,{'peer_id':peer_id}),
'Nodes-local' : Nodes (self.api,{'peer_id':None}),
- 'Nodes-peer' : Nodes (self.api,{'~peer_id':None}),
+ 'Nodes-peer' : Nodes (self.api,{'peer_id':peer_id}),
'Persons-local' : Persons (self.api,{'peer_id':None}),
- 'Persons-peer' : Persons (self.api,{'~peer_id':None}),
+ 'Persons-peer' : Persons (self.api,{'peer_id':peer_id}),
'SliceAttibuteTypes-local' : SliceAttributeTypes (self.api,{'peer_id':None}),
- 'SliceAttibuteTypes-peer' : SliceAttributeTypes (self.api,{'~peer_id':None}),
+ 'SliceAttibuteTypes-peer' : SliceAttributeTypes (self.api,{'peer_id':peer_id}),
'Slices-local' : Slices (self.api,{'peer_id':None}),
- 'Slices-peer' : Slices (self.api,{'~peer_id':None}),
+ 'Slices-peer' : Slices (self.api,{'peer_id':peer_id}),
'SliceAttributes-local': SliceAttributes (self.api,{'peer_id':None}),
- 'SliceAttributes-peer': SliceAttributes (self.api,{'~peer_id':None}),
+ 'SliceAttributes-peer': SliceAttributes (self.api,{'peer_id':peer_id}),
}
t_end = time.time()
result['ellapsed'] = t_end-t_start
--- /dev/null
+from PLC.Method import Method
+from PLC.Parameter import Parameter
+from PLC.Auth import Auth
+
+from PLC.Peers import Peer, Peers
+
+class GetPeerName (Method):
+ """
+ Returns this peer's name, as defined in the config as PLC_NAME
+ """
+ roles = ['admin']
+
+ accepts = [Auth()]
+
+ returns = Parameter (Peer.fields['peername'])
+
+ def call (self, auth):
+ return self.api.config.PLC_NAME
+
else:
all_nodes = Nodes(self.api, node_filter).dict()
- # Get default slices
- system_slice_attributes = SliceAttributes(self.api, {'name': 'system', 'value': '1'}).dict()
+ # Get local default slices
+ system_slice_attributes = SliceAttributes(self.api, {'name': 'system', 'value': '1', 'peer_id': None}).dict()
system_slice_ids = [slice_attribute['slice_id'] for slice_attribute in system_slice_attributes.values()]
system_slice_ids = dict.fromkeys(system_slice_ids)
class RefreshPeer(Method):
"""
- Query a peer PLC for its list of nodes, and refreshes
- the local database accordingly
-
+ First queries a remote PLC for its name and updates the local peers table
+
+ Then proceeds with fetching objects on the remote PLC, and updates the
+ local database accordingly
+
+ It requires the remote peer to be aware of our own name (as configured in PLC_NAME)
+
Returns a dict containing
- (*) 'plcname' : the peer name
- (*) 'new_sites': the number of new sites from that peer - may be negative
- (*) 'new_keys':
- (*) 'new_nodes':
- (*) 'new_persons':
- (*) 'new_slices':
+ (*) 'peername' : the peer's name
+ (*) 'new_xxx': the number of new objects from that peer - may be negative
+ (*) 'timers': various stats on performance for optimization
+
"""
roles = ['admin']
accepts = [ Auth(),
- Parameter (int, "Peer id"),
+ Mixed(Peer.fields['peer_id'],
+ Peer.fields['peername']),
]
- returns = Parameter(dict, "plcname, new_sites, new_keys, new_nodes, new_persons, new_slices")
+ returns = Parameter(dict,
+ 'new_sites '
+ 'new_keys '
+ 'new_nodes '
+ 'new_persons '
+ 'new_slice_attribute_types '
+ 'new_slices '
+ 'new_slice_attributes '
+ 'timers ')
- def call (self, auth, peer_id):
+ def call (self, auth, peer_id_or_peername):
### retrieve peer info
- peers = Peers (self.api,[peer_id])
+ peers = Peers (self.api,[peer_id_or_peername])
try:
peer=peers[0]
except:
- raise PLCInvalidArgument,'no such peer_id:%d'%peer_id
+ raise PLCInvalidArgument,'RefreshPeer: no such peer:%r'%peer_id_or_peername
### retrieve account info
- person_id = peer['person_id']
- persons = Persons (self.api,[person_id])
+ auth_person_id = peer['auth_person_id']
+ persons = Persons (self.api,[auth_person_id])
try:
person = persons[0]
except:
- raise PLCInvalidArgument,'no such person_id:%d'%person_id
+ raise PLCInvalidArgument,'RefreshPeer: no such person_id:%d'%auth_person_id
## connect to the peer's API
url=peer['peer_url']
auth={ 'Username': person['email'],
'AuthMethod' : 'password',
'AuthString' : person['password'],
- 'Role' : 'admin' }
+ 'Role' : 'admin' ,
+ }
+
+ # xxx
+ # right now we *need* the remote peer to know our name
+ # (this is used in the GetPeerData that we issue)
+ # in general this will be true
+ # however if anyone decides to change its own plc name things can get wrong
+ # doing this should ensure things become right again after some iterations
+ # that is, hopefully
+ # might wish to change this policy once we have peer authentication, maybe
+
+ # make sure we have the right name for that peer
+ peername = apiserver.GetPeerName (auth)
+ peer.update_name(peername)
+
+ # we need a peer_id from there on
+ peer_id = peer['peer_id']
+
+ print 'Got peer_id',peer_id
cache = Cache (self.api, peer_id, apiserver, auth)
- return cache.refresh_peer ()
+ result = cache.refresh_peer ()
+ result['peername']=peername
+
+ return result
-methods = 'AddAddressType AddAddressTypeToAddress AddBootState AddConfFile AddConfFileToNodeGroup AddConfFileToNode AddKeyType AddMessage AddNetworkMethod AddNetworkType AddNodeGroup AddNodeNetwork AddNode AddNodeToNodeGroup AddNodeToPCU AddPCU AddPeer AddPersonKey AddPerson AddPersonToSite AddPersonToSlice AddRole AddRoleToPerson AddSiteAddress AddSite AddSliceAttribute AddSliceAttributeType AddSliceInstantiation AddSlice AddSliceToNodes AdmAddAddressType AdmAddNodeGroup AdmAddNodeNetwork AdmAddNode AdmAddNodeToNodeGroup AdmAddPersonKey AdmAddPerson AdmAddPersonToSite AdmAddSitePowerControlUnit AdmAddSite AdmAssociateNodeToPowerControlUnitPort AdmAuthCheck AdmDeleteAddressType AdmDeleteAllPersonKeys AdmDeleteNodeGroup AdmDeleteNodeNetwork AdmDeleteNode AdmDeletePersonKeys AdmDeletePerson AdmDeleteSitePowerControlUnit AdmDeleteSite AdmDisassociatePowerControlUnitPort AdmGenerateNodeConfFile AdmGetAllAddressTypes AdmGetAllKeyTypes AdmGetAllNodeNetworks AdmGetAllRoles AdmGetNodeGroupNodes AdmGetNodeGroups AdmGetNodes AdmGetPersonKeys AdmGetPersonRoles AdmGetPersonSites AdmGetPersons AdmGetPowerControlUnitNodes AdmGetPowerControlUnits AdmGetSiteNodes AdmGetSitePersons AdmGetSitePIs AdmGetSitePowerControlUnits AdmGetSites AdmGetSiteTechContacts AdmGrantRoleToPerson AdmIsPersonInRole AdmQueryConfFile AdmQueryNode AdmQueryPerson AdmQueryPowerControlUnit AdmQuerySite AdmRebootNode AdmRemoveNodeFromNodeGroup AdmRemovePersonFromSite AdmRevokeRoleFromPerson AdmSetPersonEnabled AdmSetPersonPrimarySite AdmUpdateNodeGroup AdmUpdateNodeNetwork AdmUpdateNode AdmUpdatePerson AdmUpdateSitePowerControlUnit AdmUpdateSite AuthCheck BlacklistKey BootCheckAuthentication BootGetNodeDetails BootNotifyOwners BootUpdateNode DeleteAddress DeleteAddressTypeFromAddress DeleteAddressType DeleteBootState DeleteConfFileFromNodeGroup DeleteConfFileFromNode DeleteConfFile DeleteKey DeleteKeyType DeleteMessage DeleteNetworkMethod DeleteNetworkType DeleteNodeFromNodeGroup DeleteNodeFromPCU DeleteNodeGroup DeleteNodeNetwork DeleteNode DeletePCU DeletePersonFromSite DeletePersonFromSlice DeletePerson DeleteRoleFromPerson DeleteRole DeleteSession DeleteSite DeleteSliceAttribute DeleteSliceAttributeType DeleteSliceFromNodes DeleteSliceInstantiation DeleteSlice GetAddresses GetAddressTypes GetBootStates GetConfFiles GetEvents GetKeys GetKeyTypes GetMessages GetNetworkMethods GetNetworkTypes GetNodeGroups GetNodeNetworks GetNodes GetPCUs GetPeerData GetPeers GetPersons GetRoles GetSession GetSites GetSliceAttributes GetSliceAttributeTypes GetSliceInstantiations GetSlices GetSlivers RebootNode RefreshPeer SetPersonPrimarySite SliceCreate SliceDelete UpdateAddress UpdateAddressType UpdateConfFile UpdateKey UpdateMessage UpdateNodeGroup UpdateNodeNetwork UpdateNode UpdatePCU UpdatePerson UpdateSite UpdateSliceAttribute UpdateSliceAttributeType UpdateSlice system.listMethods system.methodHelp system.methodSignature system.multicall'.split()
+methods = 'AddAddressType AddAddressTypeToAddress AddBootState AddConfFile AddConfFileToNodeGroup AddConfFileToNode AddKeyType AddMessage AddNetworkMethod AddNetworkType AddNodeGroup AddNodeNetwork AddNode AddNodeToNodeGroup AddNodeToPCU AddPCU AddPeer AddPersonKey AddPerson AddPersonToSite AddPersonToSlice AddRole AddRoleToPerson AddSiteAddress AddSite AddSliceAttribute AddSliceAttributeType AddSliceInstantiation AddSlice AddSliceToNodes AdmAddAddressType AdmAddNodeGroup AdmAddNodeNetwork AdmAddNode AdmAddNodeToNodeGroup AdmAddPersonKey AdmAddPerson AdmAddPersonToSite AdmAddSitePowerControlUnit AdmAddSite AdmAssociateNodeToPowerControlUnitPort AdmAuthCheck AdmDeleteAddressType AdmDeleteAllPersonKeys AdmDeleteNodeGroup AdmDeleteNodeNetwork AdmDeleteNode AdmDeletePersonKeys AdmDeletePerson AdmDeleteSitePowerControlUnit AdmDeleteSite AdmDisassociatePowerControlUnitPort AdmGenerateNodeConfFile AdmGetAllAddressTypes AdmGetAllKeyTypes AdmGetAllNodeNetworks AdmGetAllRoles AdmGetNodeGroupNodes AdmGetNodeGroups AdmGetNodes AdmGetPersonKeys AdmGetPersonRoles AdmGetPersonSites AdmGetPersons AdmGetPowerControlUnitNodes AdmGetPowerControlUnits AdmGetSiteNodes AdmGetSitePersons AdmGetSitePIs AdmGetSitePowerControlUnits AdmGetSites AdmGetSiteTechContacts AdmGrantRoleToPerson AdmIsPersonInRole AdmQueryConfFile AdmQueryNode AdmQueryPerson AdmQueryPowerControlUnit AdmQuerySite AdmRebootNode AdmRemoveNodeFromNodeGroup AdmRemovePersonFromSite AdmRevokeRoleFromPerson AdmSetPersonEnabled AdmSetPersonPrimarySite AdmUpdateNodeGroup AdmUpdateNodeNetwork AdmUpdateNode AdmUpdatePerson AdmUpdateSitePowerControlUnit AdmUpdateSite AuthCheck BlacklistKey BootCheckAuthentication BootGetNodeDetails BootNotifyOwners BootUpdateNode DeleteAddress DeleteAddressTypeFromAddress DeleteAddressType DeleteBootState DeleteConfFileFromNodeGroup DeleteConfFileFromNode DeleteConfFile DeleteKey DeleteKeyType DeleteMessage DeleteNetworkMethod DeleteNetworkType DeleteNodeFromNodeGroup DeleteNodeFromPCU DeleteNodeGroup DeleteNodeNetwork DeleteNode DeletePCU DeletePersonFromSite DeletePersonFromSlice DeletePerson DeleteRoleFromPerson DeleteRole DeleteSession DeleteSite DeleteSliceAttribute DeleteSliceAttributeType DeleteSliceFromNodes DeleteSliceInstantiation DeleteSlice GetAddresses GetAddressTypes GetBootStates GetConfFiles GetEvents GetKeys GetKeyTypes GetMessages GetNetworkMethods GetNetworkTypes GetNodeGroups GetNodeNetworks GetNodes GetPCUs GetPeerData GetPeerName GetPeers GetPersons GetRoles GetSession GetSites GetSliceAttributes GetSliceAttributeTypes GetSliceInstantiations GetSlices GetSlivers RebootNode RefreshPeer SetPersonPrimarySite SliceCreate SliceDelete UpdateAddress UpdateAddressType UpdateConfFile UpdateKey UpdateMessage UpdateNodeGroup UpdateNodeNetwork UpdateNode UpdatePCU UpdatePerson UpdateSite UpdateSliceAttribute UpdateSliceAttributeType UpdateSlice system.listMethods system.methodHelp system.methodSignature system.multicall'.split()
'peer_id' : Parameter (int, "Peer identifier"),
'peername' : Parameter (str, "Peer name"),
'peer_url' : Parameter (str, "Peer API url"),
- 'person_id' : Parameter (int, "Person_id of the account storing credentials - temporary"),
+ ### xxx this trick is temporary, for peer authentication
+ 'auth_person_id' : Parameter (int, "Person_id of the account storing credentials - temporary"),
+ ### cross refs
+ 'site_ids' : Parameter ([int], "This peer's sites ids"),
+ 'person_ids' : Parameter ([int], "This peer's persons ids"),
'node_ids' : Parameter ([int], "This peer's nodes ids"),
'slice_ids' : Parameter ([int], "This peer's slices ids"),
}
raise invalid_url
return url
+ ### for use by RefreshPeer, *not* a method of the API
+ def update_name (self,peername):
+ if self['peername'] != peername:
+ self['peername']=peername
+ self.sync()
+
def delete (self, commit=True):
"""
Delete peer
--
-- Copyright (C) 2006 The Trustees of Princeton University
--
--- $Id: planetlab4.sql,v 1.51 2006/11/28 22:00:14 tmack Exp $
+-- $Id: planetlab4.sql,v 1.52 2006/11/29 17:57:27 tmack Exp $
--
--------------------------------------------------------------------------------
peer_url text NOT NULL, -- the url of that peer's API
-- oops, looks like we have a dependency loop here
--person_id integer REFERENCES persons NOT NULL, -- the account we use for logging in
- person_id integer NOT NULL, -- the account we use for logging in
+ auth_person_id integer NOT NULL, -- the account we use for logging in
deleted boolean NOT NULL DEFAULT false
) WITH OIDS;
model text, -- Hardware make and model
boot_nonce text, -- Random nonce updated by Boot Manager
version text, -- Boot CD version string updated by Boot Manager
- -- XXX Should be key_id integer REFERENCES keys
ssh_rsa_key text, -- SSH host key updated by Boot Manager
key text, -- Node key generated by API when configuration file is downloaded
LEFT JOIN person_keys USING (person_id)
LEFT JOIN person_slices USING (person_id);
--- Nodes at each peer
+-- Objects at each peer
+CREATE VIEW peer_sites AS
+SELECT peer_id,
+array_accum(site_id) AS site_ids
+FROM sites
+GROUP BY peer_id;
+
+CREATE VIEW peer_persons AS
+SELECT peer_id,
+array_accum(person_id) AS person_ids
+FROM persons
+GROUP BY peer_id;
+
CREATE VIEW peer_nodes AS
SELECT peer_id,
array_accum(node_id) AS node_ids
CREATE VIEW view_peers AS
SELECT
peers.*,
+peer_sites.site_ids,
+peer_persons.person_ids,
peer_nodes.node_ids,
peer_slices.slice_ids
FROM peers
+LEFT JOIN peer_sites USING (peer_id)
+LEFT JOIN peer_persons USING (peer_id)
LEFT JOIN peer_nodes USING (peer_id)
LEFT JOIN peer_slices USING (peer_id);
git://git.onelab.eu / plcapi.git / commitdiff