From 2696eec3582eb1b1092d0a98c863a040f6aa2af4 Mon Sep 17 00:00:00 2001 From: Thierry Parmentelat Date: Tue, 21 Nov 2006 10:57:00 +0000 Subject: [PATCH] iteration 4 & last: - ForeignNode(s) and ForeignSlice(s) deprecated - Node and Slice both have a peer_id field that can be used for testing locality (peer_is is None) - associations peer x node and peer x slice not managed in a separate table anymore (were peer_node and peer_slice) peer_id is defined as a native column instead - local nodes can be retrieved with GetNodes (auth, {'peer_id':None}) GetNodes (auth, {'hostname':[name1,name2], 'peer_id':None}) or GetNodes (auth, None, None, 'local') GetNodes (auth, ['name1','name2'] , None, 'local') - foreign nodes can be retrieved similarly with GetNodes (auth, {'hostname':[name1,name2], '~peer_id':None}) or GetNodes (auth, ['name1','name2'] , None, 'foreign') - the 4th argument to GetNodes is named 'scope'. It is only experimental by now, Slices do not have it as of now, will decide later whether we provide it for all cached entities - WARNING : Having Nodes and Slices return ALL entities might have impacts on other methods; most of which making sense on LOCAL entities only. Might need a review. --- PLC/ForeignNodes.py | 77 ----------------- PLC/ForeignSlices.py | 108 ------------------------ PLC/Methods/AddNode.py | 3 + PLC/Methods/AddSliceToNodes.py | 10 +-- PLC/Methods/DeleteSliceFromNodes.py | 7 +- PLC/Methods/GetForeignNodes.py | 36 -------- PLC/Methods/GetForeignSlices.py | 36 -------- PLC/Methods/GetNodes.py | 7 +- PLC/Methods/GetSlivers.py | 5 +- PLC/Methods/RefreshPeer.py | 15 ++-- PLC/Nodes.py | 18 +++- PLC/Peers.py | 123 ++++++---------------------- PLC/Slices.py | 37 ++++++++- TestPeers.py | 59 ++++++++----- peers-test.mk | 6 +- planetlab4.sql | 97 ++++++---------------- 16 files changed, 160 insertions(+), 484 deletions(-) delete mode 100644 PLC/ForeignNodes.py delete mode 100644 PLC/ForeignSlices.py delete mode 100644 PLC/Methods/GetForeignNodes.py delete mode 100644 PLC/Methods/GetForeignSlices.py diff --git a/PLC/ForeignNodes.py b/PLC/ForeignNodes.py deleted file mode 100644 index 2f25b61..0000000 --- a/PLC/ForeignNodes.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# Thierry Parmentelat - INRIA -# -import time - -from types import StringTypes - -from PLC.Table import Row, Table -from PLC.Parameter import Parameter -from PLC.Filter import Filter - -class ForeignNode (Row) : - """ - This object stores information about nodes hosted on - other peering instances of myplc - """ - - table_name = 'nodes' - primary_key = 'node_id' - - fields = { - 'node_id': Parameter (int, "Node Id"), - 'hostname': Parameter (str, "Node name"), - 'peer_id': Parameter (str, "Peer id"), - 'boot_state' : Parameter (str, "Boot state, see Node"), - 'model' : Parameter (str,"Model, see Node"), - 'version' : Parameter (str,"Version, see Node"), - 'date_created': Parameter(int, "Creation time, see Node"), - 'last_updated': Parameter(int, "Update time, see Node"), - 'slice_ids': Parameter([int], "List of slices on this node"), - } - - def __init__(self,api,fields={},uptodate=True): - Row.__init__(self,api,fields) - self.uptodate=uptodate - - def validate_date_created(self,timestamp): - return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(timestamp)) - - def validate_last_updated(self,timestamp): - return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(timestamp)) - - def purge_peer_node (self,commit=True): - sql = "DELETE FROM peer_node WHERE node_id=%d"%self['node_id'] - self.api.db.do(sql) - if commit: - self.api.db.commit() - - def delete (self, commit=True): - """ - Delete existing foreign node. - """ - self.purge_peer_node() - self['deleted']=True - self.sync(commit) - -class ForeignNodes (Table): - def __init__ (self, api, foreign_node_filter = None, columns = None): - Table.__init__(self, api, ForeignNode, columns) - - sql = "" - sql += "SELECT %s FROM view_foreign_nodes " % ", ".join(self.columns) - sql += "WHERE deleted IS False " - - if foreign_node_filter is not None: - if isinstance(foreign_node_filter, (list, tuple, set)): - # Separate the list into integers and strings - ints = filter(lambda x: isinstance(x, (int, long)), foreign_node_filter) - strs = filter(lambda x: isinstance(x, StringTypes), foreign_node_filter) - foreign_node_filter = Filter(ForeignNode.fields, {'node_id': ints, 'hostname': strs}) - sql += " AND (%s)" % foreign_node_filter.sql(api, "OR") - elif isinstance(foreign_node_filter, dict): - foreign_node_filter = Filter(ForeignNode.fields, foreign_node_filter) - sql += " AND (%s)" % foreign_node_filter.sql(api, "AND") - - self.selectall(sql) - diff --git a/PLC/ForeignSlices.py b/PLC/ForeignSlices.py deleted file mode 100644 index 317b036..0000000 --- a/PLC/ForeignSlices.py +++ /dev/null @@ -1,108 +0,0 @@ -# -# Thierry Parmentelat - INRIA -# -import time - -from types import StringTypes - -from PLC.Table import Row, Table -from PLC.Parameter import Parameter -from PLC.Filter import Filter - -class ForeignSlice (Row) : - """ - This object stores information about slices hosted on - other peering instances of myplc - """ - - table_name = 'slices' - primary_key = 'slice_id' - - fields = { - 'slice_id': Parameter (int, "Slice Id"), - 'name' : Parameter (str, "Slice name"), - 'peer_id': Parameter (str, "Peer id"), - 'instantiation': Parameter(str, "Slice instantiation state"), - 'url': Parameter(str, "URL further describing this slice", max = 254, nullok = True), - 'description': Parameter(str, "Slice description", max = 2048, nullok = True), - 'max_nodes': Parameter(int, "Maximum number of nodes that can be assigned to this slice"), - 'created': Parameter(int, "Date and time when slice was created, in seconds since UNIX epoch", ro = True), - 'expires': Parameter(int, "Date and time when slice expires, in seconds since UNIX epoch"), - 'node_ids' : Parameter([int], "List of nodes in this slice"), - } - - def __init__(self,api,fields={},uptodate=True): - Row.__init__(self,api,fields) - self.uptodate=uptodate - - def validate_created(self,timestamp): - return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(timestamp)) - - def validate_expires(self,timestamp): - return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(timestamp)) - - def purge_peer_slice (self,commit=True): - sql = "DELETE FROM peer_slice WHERE slice_id=%d"%self['slice_id'] - self.api.db.do(sql) - if commit: - self.api.db.commit() - - def purge_slice_node (self,commit=True): - sql = "DELETE FROM slice_node WHERE slice_id=%d"%self['slice_id'] - self.api.db.do(sql) - if commit: - self.api.db.commit() - - def add_slice_nodes (self, node_ids, commit=True): - slice_id = self['slice_id'] - ### xxx needs to be optimized - ### tried to figure a way to use a single sql statement - ### like: insert into table (x,y) values (1,2),(3,4); - ### but apparently this is not supported under postgresql - for node_id in node_ids: - sql="INSERT INTO slice_node VALUES (%d,%d)"%(slice_id,node_id) - self.api.db.do(sql) - if commit: - self.api.db.commit() - - def update_slice_nodes (self, node_ids): - # xxx to be optimized - # we could compute the (set) difference between - # current and updated set of node_ids - # and invoke the DB only based on that - # - # for now : clean all entries for this slice - self.purge_slice_node() - # and re-install new list - self.add_slice_nodes (node_ids) - - def delete (self, commit=True): - """ - Delete existing foreign slice. - """ - self.purge_peer_slice() - self['is_deleted']=True - self.sync(commit) - - -class ForeignSlices (Table): - def __init__ (self, api, foreign_slice_filter = None, columns = None): - Table.__init__(self, api, ForeignSlice, columns) - - sql = "" - sql += "SELECT %s FROM view_foreign_slices " % ", ".join(self.columns) - sql += "WHERE is_deleted IS False " - - if foreign_slice_filter is not None: - if isinstance(foreign_slice_filter, (list, tuple, set)): - # Separate the list into integers and strings - ints = filter(lambda x: isinstance(x, (int, long)), foreign_slice_filter) - strs = filter(lambda x: isinstance(x, StringTypes), foreign_slice_filter) - foreign_slice_filter = Filter(ForeignSlice.fields, {'slice_id': ints, 'name': strs}) - sql += " AND (%s)" % foreign_slice_filter.sql(api, "OR") - elif isinstance(foreign_slice_filter, dict): - foreign_slice_filter = Filter(ForeignSlice.fields, foreign_slice_filter) - sql += " AND (%s)" % foreign_slice_filter.sql(api, "AND") - - self.selectall(sql) - diff --git a/PLC/Methods/AddNode.py b/PLC/Methods/AddNode.py index 2fc9837..cacfdb2 100644 --- a/PLC/Methods/AddNode.py +++ b/PLC/Methods/AddNode.py @@ -60,6 +60,9 @@ class AddNode(Method): node = Node(self.api, node_fields) node['site_id'] = site['site_id'] + # xxx temporary, until I figure a way to get local/foreign nodes + # so far I cannot seem to use a filter/dict like {'~peer_id':None} + # node['peer_id'] = 0 node.sync() self.object_ids = [site['site_id'], node['node_id']] diff --git a/PLC/Methods/AddSliceToNodes.py b/PLC/Methods/AddSliceToNodes.py index dffb59c..1b2942c 100644 --- a/PLC/Methods/AddSliceToNodes.py +++ b/PLC/Methods/AddSliceToNodes.py @@ -2,15 +2,13 @@ from PLC.Faults import * from PLC.Method import Method from PLC.Parameter import Parameter, Mixed from PLC.Nodes import Node, Nodes -from PLC.ForeignNodes import ForeignNode, ForeignNodes from PLC.Slices import Slice, Slices from PLC.Auth import Auth class AddSliceToNodes(Method): """ Adds the specified slice to the specified nodes. - Nodes can be either regular (local) nodes as returned by GetNodes - or foreign nodes as returned by GetForeignNodes + Nodes can be either local or foreign nodes, as returned by GetNodes If the slice is already associated with a node, no errors are returned. @@ -54,14 +52,12 @@ class AddSliceToNodes(Method): # Get specified nodes, add them to the slice nodes = Nodes(self.api, node_id_or_hostname_list) - foreign_nodes = ForeignNodes (self.api, node_id_or_hostname_list) - all_nodes = nodes+foreign_nodes; - for node in all_nodes: + for node in nodes: if slice['slice_id'] not in node['slice_ids']: slice.add_node(node, commit = False) slice.sync() - self.object_ids = [node['node_id'] for node in all_nodes] + self.object_ids = [node['node_id'] for node in nodes] return 1 diff --git a/PLC/Methods/DeleteSliceFromNodes.py b/PLC/Methods/DeleteSliceFromNodes.py index d3e31f0..ce46929 100644 --- a/PLC/Methods/DeleteSliceFromNodes.py +++ b/PLC/Methods/DeleteSliceFromNodes.py @@ -2,7 +2,6 @@ from PLC.Faults import * from PLC.Method import Method from PLC.Parameter import Parameter, Mixed from PLC.Nodes import Node, Nodes -from PLC.ForeignNodes import ForeignNode, ForeignNodes from PLC.Slices import Slice, Slices from PLC.Auth import Auth @@ -49,14 +48,12 @@ class DeleteSliceFromNodes(Method): # Get specified nodes nodes = Nodes(self.api, node_id_or_hostname_list) - foreign_nodes = ForeignNodes(self.api, node_id_or_hostname_list) - all_nodes = nodes+foreign_nodes; - for node in all_nodes: + for node in nodes: if slice['slice_id'] in node['slice_ids']: slice.remove_node(node, commit = False) slice.sync() - self.object_ids = [node['node_id'] for node in all_nodes] + self.object_ids = [node['node_id'] for node in nodes] return 1 diff --git a/PLC/Methods/GetForeignNodes.py b/PLC/Methods/GetForeignNodes.py deleted file mode 100644 index f8f2fb1..0000000 --- a/PLC/Methods/GetForeignNodes.py +++ /dev/null @@ -1,36 +0,0 @@ -# -# Thierry Parmentelat - INRIA -# - -from PLC.Faults import * -from PLC.Method import Method -from PLC.Parameter import Parameter, Mixed -from PLC.Filter import Filter -from PLC.Auth import Auth - -from PLC.ForeignNodes import ForeignNode, ForeignNodes - -class GetForeignNodes(Method): - """ - Returns an array of structs containing details about foreign - nodes. If foreign_node_filter is specified and is an array of - foreign node identifiers or hostnames, or a struct of foreign node - attributes, only foreign nodes matching the filter will be - returned. If return_fields is specified, only the specified - details will be returned. - """ - - roles = ['admin'] - - accepts = [ - Auth(), - Mixed([Mixed(ForeignNode.fields['node_id'], - ForeignNode.fields['hostname'])], - Filter(ForeignNode.fields)), - Parameter([str], "List of fields to return", nullok = True) - ] - - returns = [ForeignNode.fields] - - def call(self, auth, foreign_node_filter = None, return_fields = None): - return ForeignNodes(self.api, foreign_node_filter, return_fields) diff --git a/PLC/Methods/GetForeignSlices.py b/PLC/Methods/GetForeignSlices.py deleted file mode 100644 index c1aa271..0000000 --- a/PLC/Methods/GetForeignSlices.py +++ /dev/null @@ -1,36 +0,0 @@ -# -# Thierry Parmentelat - INRIA -# - -from PLC.Faults import * -from PLC.Method import Method -from PLC.Parameter import Parameter, Mixed -from PLC.Filter import Filter -from PLC.Auth import Auth - -from PLC.ForeignSlices import ForeignSlice, ForeignSlices - -class GetForeignSlices(Method): - """ - Returns an array of structs containing details about foreign - slices. If foreign_slice_filter is specified and is an array of - foreign slice identifiers or names, or a struct of foreign slice - attributes, only foreign slices matching the filter will be - returned. If return_fields is specified, only the specified - details will be returned. - """ - - roles = ['admin'] - - accepts = [ - Auth(), - Mixed([Mixed(ForeignSlice.fields['slice_id'], - ForeignSlice.fields['name'])], - Filter(ForeignSlice.fields)), - Parameter([str], "List of fields to return", nullok = True) - ] - - returns = [ForeignSlice.fields] - - def call(self, auth, foreign_slice_filter = None, return_fields = None): - return ForeignSlices(self.api, foreign_slice_filter, return_fields) diff --git a/PLC/Methods/GetNodes.py b/PLC/Methods/GetNodes.py index 22b8e01..937b948 100644 --- a/PLC/Methods/GetNodes.py +++ b/PLC/Methods/GetNodes.py @@ -23,14 +23,15 @@ class GetNodes(Method): Mixed([Mixed(Node.fields['node_id'], Node.fields['hostname'])], Filter(Node.fields)), - Parameter([str], "List of fields to return", nullok = True) + Parameter([str], "List of fields to return", nullok = True), + Parameter(str,"scope string, can be either 'all', 'local' or 'foreign'"), ] returns = [Node.fields] - def call(self, auth, node_filter = None, return_fields = None): + def call(self, auth, node_filter = None, return_fields = None, scope = 'all'): # Get node information - nodes = Nodes(self.api, node_filter, return_fields) + nodes = Nodes(self.api, node_filter, return_fields, scope) # Remove admin only fields if 'admin' not in self.caller['roles']: diff --git a/PLC/Methods/GetSlivers.py b/PLC/Methods/GetSlivers.py index ca2bc3f..1c9746c 100644 --- a/PLC/Methods/GetSlivers.py +++ b/PLC/Methods/GetSlivers.py @@ -10,7 +10,7 @@ from PLC.NodeNetworks import NodeNetwork, NodeNetworks from PLC.NodeGroups import NodeGroup, NodeGroups from PLC.ConfFiles import ConfFile, ConfFiles from PLC.Slices import Slice, Slices -from PLC.ForeignSlices import ForeignSlice, ForeignSlices +#from PLC.ForeignSlices import ForeignSlice, ForeignSlices from PLC.Persons import Person, Persons from PLC.Keys import Key, Keys from PLC.SliceAttributes import SliceAttribute, SliceAttributes @@ -68,7 +68,6 @@ class GetSlivers(Method): all_nodes = {self.caller['node_id']: self.caller} else: all_nodes = Nodes(self.api, node_filter).dict() - # XXX Add foreign nodes # Get default slices system_slice_attributes = SliceAttributes(self.api, {'name': 'system', 'value': '1'}).dict() @@ -94,8 +93,6 @@ class GetSlivers(Method): # Get slice information all_slices = Slices(self.api, all_slice_ids).dict() - # XXX Merge in foreign slices - all_slices.update(ForeignSlices(self.api, all_slice_ids).dict()) person_ids = set() slice_attribute_ids = set() diff --git a/PLC/Methods/RefreshPeer.py b/PLC/Methods/RefreshPeer.py index 8f4aaef..fa80f11 100644 --- a/PLC/Methods/RefreshPeer.py +++ b/PLC/Methods/RefreshPeer.py @@ -11,7 +11,6 @@ from PLC.Auth import Auth from PLC.Peers import Peer, Peers from PLC.Persons import Person, Persons -##from PLC.ForeignNodes import ForeignNode, ForeignNodes class RefreshPeer(Method): @@ -57,17 +56,15 @@ class RefreshPeer(Method): ## connect to the peer's API url=peer['peer_url'] - print 'url=',url - apiserver = xmlrpclib.Server (url) - print 'auth=',auth + apiserver = xmlrpclib.ServerProxy (url,allow_none=True) - peer_get_nodes = apiserver.GetNodes(auth) - nb_new_nodes = peer.refresh_nodes(peer_get_nodes) + peer_local_nodes = apiserver.GetNodes(auth,None,None,'local') + nb_new_nodes = peer.refresh_nodes(peer_local_nodes) # rough and temporary - peer_foreign_nodes = apiserver.GetForeignNodes(auth) - peer_get_slices = apiserver.GetSlices(auth) - nb_new_slices = peer.refresh_slices(peer_get_slices,peer_foreign_nodes) + peer_foreign_nodes = apiserver.GetNodes(auth,None,None,'foreign') + peer_local_slices = apiserver.GetSlices(auth,{'peer_id':None}) + nb_new_slices = peer.refresh_slices(peer_local_slices,peer_foreign_nodes) return {'plcname':self.api.config.PLC_NAME, 'new_nodes':nb_new_nodes, diff --git a/PLC/Nodes.py b/PLC/Nodes.py index ae2a5e7..bf460ca 100644 --- a/PLC/Nodes.py +++ b/PLC/Nodes.py @@ -4,7 +4,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: Nodes.py,v 1.19 2006/11/09 03:07:42 mlhuang Exp $ +# $Id: Nodes.py,v 1.20 2006/11/09 19:43:55 mlhuang Exp $ # from types import StringTypes @@ -43,6 +43,7 @@ class Node(Row): 'node_id': Parameter(int, "Node identifier"), 'hostname': Parameter(str, "Fully qualified hostname", max = 255), 'site_id': Parameter(int, "Site at which this node is located"), + 'peer_id': Parameter(int, "Peer at which this node is managed", nullok = True), 'boot_state': Parameter(str, "Boot state", max = 20), 'model': Parameter(str, "Make and model of the actual machine", max = 255, nullok = True), 'boot_nonce': Parameter(str, "(Admin only) Random value generated by the node at last boot", max = 128), @@ -61,6 +62,12 @@ class Node(Row): 'ports': Parameter([int], "List of PCU ports that this node is connected to"), } + # foreign attributes management + # the key to track remote objects + foreign_key = 'hostname' + # the fields that get verbatim copied from foreign objects + foreign_fields = ['boot_state','model','version','date_created','last_updated'] + def validate_hostname(self, hostname): if not valid_hostname(hostname): raise PLCInvalidArgument, "Invalid hostname" @@ -95,13 +102,14 @@ class Node(Row): self['deleted'] = True self.sync(commit) + class Nodes(Table): """ Representation of row(s) from the nodes table in the database. """ - def __init__(self, api, node_filter = None, columns = None): + def __init__(self, api, node_filter = None, columns = None, scope = 'all'): Table.__init__(self, api, Node, columns) sql = "SELECT %s FROM view_nodes WHERE deleted IS False" % \ @@ -118,4 +126,10 @@ class Nodes(Table): node_filter = Filter(Node.fields, node_filter) sql += " AND (%s)" % node_filter.sql(api, "AND") + if scope == 'local': + sql += " AND (peer_id is NULL) " + elif scope == 'foreign': + sql += " AND (peer_id is NOT NULL) " + self.selectall(sql) + diff --git a/PLC/Peers.py b/PLC/Peers.py index c4ff3df..db049b0 100644 --- a/PLC/Peers.py +++ b/PLC/Peers.py @@ -11,8 +11,7 @@ from PLC.Filter import Filter from PLC.Table import Row, Table from PLC.Nodes import Nodes,Node -from PLC.ForeignNodes import ForeignNodes,ForeignNode -from PLC.ForeignSlices import ForeignSlices,ForeignSlice +from PLC.Slices import Slices,Slice class Peer (Row): """ @@ -48,90 +47,21 @@ class Peer (Row): assert 'peer_id' in self # remove nodes depending on this peer - for foreign_node_id in self.get_foreign_nodes(): - try: - foreign_node = ForeignNodes(self.api,[foreign_node_id])[0] - foreign_node.delete(commit) - except: - print "Glitch : a foreign node instance was uncleanly deleted" + for foreign_node in Nodes (self.api, self.get_node_ids()): + foreign_node.delete(commit) # remove the peer self['deleted'] = True self.sync(commit) - def get_foreign_nodes (self): + def get_node_ids (self): """ - returns a list of the foreign nodes in this peer + returns a list of the node ids in this peer """ sql="SELECT node_ids FROM peer_nodes WHERE peer_id=%d"%self['peer_id'] node_ids = self.api.db.selectall(sql) return node_ids[0]['node_ids'] - def manage_node (self, foreign_node, add_if_true, commit=True): - """ - associate/dissociate a foreign node to/from a peer - foreign_node is a local object that describes a remote node - convention is: - if add_if_true is None : performs dissociation - otherwise: performs association - """ - - assert 'peer_id' in self - assert 'node_id' in foreign_node - - peer_id = self['peer_id'] - node_id = foreign_node ['node_id'] - - if add_if_true: - ### ADDING - sql = "INSERT INTO peer_node VALUES (%d,%d)" % (peer_id,node_id) - self.api.db.do(sql) - if self['node_ids'] is None: - self['node_ids']=[node_id,] - else: - self['node_ids'].append(node_id) - ### DELETING - else: - sql = "DELETE FROM peer_node WHERE peer_id=%d AND node_id=%d" % (peer_id,node_id) - self.api.db.do(sql) - self['node_ids'].remove(node_id) - - if commit: - self.api.db.commit() - - def manage_slice (self, foreign_slice, add_if_true, commit=True): - """ - associate/dissociate a foreign node to/from a peer - foreign_slice is a local object that describes a remote slice - alien_id is the unique id as provided by the remote peer - convention is: - if add_if_true is None : performs dissociation - otherwise: performs association - """ - - assert 'peer_id' in self - assert 'slice_id' in foreign_slice - - peer_id = self['peer_id'] - slice_id = foreign_slice ['slice_id'] - - if add_if_true: - ### ADDING - sql = "INSERT INTO peer_slice VALUES (%d,%d)" % (peer_id,slice_id) - self.api.db.do(sql) - if self['slice_ids'] is None: - self['slice_ids']=[slice_id,] - else: - self['slice_ids'].append(slice_id) - ### DELETING - else: - sql = "DELETE FROM peer_slice WHERE peer_id=%d AND slice_id=%d" % (peer_id,slice_id) - self.api.db.do(sql) - self['slice_ids'].remove(slice_id) - - if commit: - self.api.db.commit() - def refresh_nodes (self, peer_get_nodes): """ refreshes the foreign_nodes and peer_node tables @@ -144,9 +74,13 @@ class Peer (Row): # we get the whole table just in case # a host would have switched from one plc to the other - local_foreign_nodes = ForeignNodes (self.api) + local_foreign_nodes = Nodes (self.api,None,None,'foreign') + # index it by hostname for searching later - local_foreign_nodes_index = local_foreign_nodes.dict('hostname') + #local_foreign_nodes_index=local_foreign_nodes.dict('hostname') + local_foreign_nodes_index={} + for node in local_foreign_nodes: + local_foreign_nodes_index[node['hostname']]=node ### mark entries for this peer outofdate old_count=0; @@ -164,14 +98,7 @@ class Peer (Row): try: foreign_node = local_foreign_nodes_index[hostname] if foreign_node['peer_id'] != peer_id: - ### the node has changed its plc, needs to update peer_node - old_peer_id = foreign_node['peer_id'] - old_peers=Peers(self.api,[peer_id]) - assert old_peer[0] - # remove from previous peer - old_peers[0].manage_node(foreign_node,False,False) - # add to new peer - self.manage_node(foreign_node,True,True) +# ### the node has changed its plc foreign_node['peer_id'] = peer_id ### update it anyway: copy other relevant fields for field in remote_fields: @@ -180,13 +107,14 @@ class Peer (Row): foreign_node.uptodate=True foreign_node.sync() except: - new_foreign_node = ForeignNode(self.api, {'hostname':hostname}) + new_foreign_node = Node(self.api, {'hostname':hostname}) + new_foreign_node['peer_id']=peer_id for field in remote_fields: new_foreign_node[field]=node[field] ### need to sync so we get a node_id new_foreign_node.sync() new_foreign_node.uptodate = True - self.manage_node(new_foreign_node,True,True) +# self.manage_node(new_foreign_node,True,True) local_foreign_nodes_index[hostname]=new_foreign_node ### delete entries that are not uptodate @@ -219,7 +147,7 @@ class Peer (Row): # we get the whole table just in case # a host would have switched from one plc to the other - local_foreign_slices = ForeignSlices (self.api) + local_foreign_slices = Slices (self.api,{'~peer_id':None}) # index it by name for searching later local_foreign_slices_index = local_foreign_slices.dict('name') @@ -237,6 +165,7 @@ class Peer (Row): ### scan the new entries, and mark them uptodate new_count=0 for slice in peer_get_slices: + ### ignore system-wide slices if slice['creator_person_id'] == 1: continue @@ -247,22 +176,15 @@ class Peer (Row): try: foreign_slice = local_foreign_slices_index[name] if foreign_slice['peer_id'] != peer_id: - ### the slice has changed its plc, needs to update peer_slice - old_peer_id = foreign_slice['peer_id'] - old_peers=Peers(self.api,[peer_id]) - assert old_peer[0] - # remove from previous peer - old_peers[0].manage_slice(foreign_slice,False,False) - # add to new peer - self.manage_slice(foreign_slice,True,True) - foreign_slice['peer_id'] = peer_id + # more suspucious ? - the slice moved on another peer + foreign_slice['peer_id'] = peer_id; except: - foreign_slice = ForeignSlice(self.api, {'name':name}) + foreign_slice = Slice(self.api, {'name':name}) + foreign_slice['peer_id']=self['peer_id'] # ### xxx temporary # foreign_slice['site_id']=1 ### need to sync so we get a slice_id foreign_slice.sync() - self.manage_slice(foreign_slice,True,True) # insert in index local_foreign_slices_index[name]=foreign_slice @@ -286,7 +208,8 @@ class Peer (Row): updated_node_ids = [] for alien_node_id in slice['node_ids']: try: - local_node_id=self.locate_alien_node_id_in_foreign_nodes(peer_foreign_nodes_dict,alien_node_id) + local_node_id=self.locate_alien_node_id_in_foreign_nodes(peer_foreign_nodes_dict, + alien_node_id) updated_node_ids.append(local_node_id) except: # this node_id is not in our scope diff --git a/PLC/Slices.py b/PLC/Slices.py index 42da1d4..56c98d9 100644 --- a/PLC/Slices.py +++ b/PLC/Slices.py @@ -9,7 +9,6 @@ from PLC.Debug import profile from PLC.Table import Row, Table from PLC.SliceInstantiations import SliceInstantiations from PLC.Nodes import Node, Nodes -from PLC.ForeignNodes import ForeignNode, ForeignNodes import PLC.Persons class Slice(Row): @@ -25,6 +24,7 @@ class Slice(Row): fields = { 'slice_id': Parameter(int, "Slice identifier"), 'site_id': Parameter(int, "Identifier of the site to which this slice belongs"), + 'peer_id': Parameter(int, "Peer at which this slice was created", nullok = True), 'name': Parameter(str, "Slice name", max = 32), 'instantiation': Parameter(str, "Slice instantiation state"), 'url': Parameter(str, "URL further describing this slice", max = 254, nullok = True), @@ -131,7 +131,7 @@ class Slice(Row): """ assert 'slice_id' in self - assert isinstance(node, (Node,ForeignNode)) + assert isinstance(node, Node) assert 'node_id' in node slice_id = self['slice_id'] @@ -156,7 +156,7 @@ class Slice(Row): """ assert 'slice_id' in self - assert isinstance(node, (Node,ForeignNode)) + assert isinstance(node, Node) assert 'node_id' in node slice_id = self['slice_id'] @@ -176,6 +176,37 @@ class Slice(Row): self['node_ids'].remove(node_id) node['slice_ids'].remove(slice_id) + ########## for foreign slices update, from ForeignSlices + def purge_slice_node (self,commit=True): + sql = "DELETE FROM slice_node WHERE slice_id=%d"%self['slice_id'] + self.api.db.do(sql) + if commit: + self.api.db.commit() + + def add_slice_nodes (self, node_ids, commit=True): + slice_id = self['slice_id'] + ### xxx needs to be optimized + ### tried to figure a way to use a single sql statement + ### like: insert into table (x,y) values (1,2),(3,4); + ### but apparently this is not supported under postgresql + for node_id in node_ids: + sql="INSERT INTO slice_node VALUES (%d,%d)"%(slice_id,node_id) + self.api.db.do(sql) + if commit: + self.api.db.commit() + + def update_slice_nodes (self, node_ids): + # xxx to be optimized + # we could compute the (set) difference between + # current and updated set of node_ids + # and invoke the DB only based on that + # + # for now : clean all entries for this slice + self.purge_slice_node() + # and re-install new list + self.add_slice_nodes (node_ids) + + ########## def sync(self, commit = True): """ Add or update a slice. diff --git a/TestPeers.py b/TestPeers.py index 82dc279..1384e9a 100755 --- a/TestPeers.py +++ b/TestPeers.py @@ -73,6 +73,9 @@ def define_test (nodes,slices): global number_nodes, number_slices number_nodes=nodes number_slices=slices + +def fast(): + define_test(1,1) define_test (nodes=5,slices=3) @@ -125,7 +128,7 @@ def test00_init (args=[1,2]): for i in args: url=plc[i]['url-format']%plc[i]['hostname'] plc[i]['url']=url - s[i]=xmlrpclib.Server(url) + s[i]=xmlrpclib.ServerProxy(url,allow_none=True) print 'initializing s[%d]'%i,url aa[i]={'Username':plc[i]['builtin_admin_id'], 'AuthMethod':'password', @@ -148,8 +151,11 @@ def test00_print (args=[1,2]): def check_nodes (en,ef,args=[1,2]): global plc,s,a for i in args: - n=len(s[i].GetNodes(a[i])) - f=len(s[i].GetForeignNodes(a[i])) + # use a single request and sort afterwards for efficiency + # could have used GetNodes's scope as well + all_nodes = s[i].GetNodes(a[i]) + n = len ([ x for x in all_nodes if x['peer_id'] is None]) + f = len ([ x for x in all_nodes if x['peer_id'] is not None]) print '%02d: Checking nodes: got %d local (e=%d) & %d foreign (e=%d)'%(i,n,en,f,ef) assert n==en assert f==ef @@ -158,15 +164,19 @@ def check_nodes (en,ef,args=[1,2]): def check_slices (els,efs,args=[1,2]): global plc,s,a for i in args: - ls=len(s[i].GetSlices(a[i])) - fs=len(s[i].GetForeignSlices(a[i])) + ls=len(s[i].GetSlices(a[i],{'peer_id':None})) + fs=len(s[i].GetSlices(a[i],{'~peer_id':None})) print '%02d: Checking slices: got %d local (e=%d) & %d foreign (e=%d)'%(i,ls,els,fs,efs) assert els==ls assert efs==fs def show_nodes (i,node_ids): - for message,nodes in [ ['LOC',s[i].GetNodes(a[i],node_ids)], - ['FOR',s[i].GetForeignNodes(a[i],node_ids)] ]: + # same as above + all_nodes = s[i].GetNodes(a[i],node_ids) + loc_nodes = filter (lambda n: n['peer_id'] is None, all_nodes) + for_nodes = filter (lambda n: n['peer_id'] is not None, all_nodes) + + for message,nodes in [ ['LOC',loc_nodes], ['FOR',for_nodes] ] : if nodes: print '[%s:%d] : '%(message,len(nodes)), for node in nodes: @@ -183,11 +193,11 @@ def check_slice_nodes_n (ns,expected_nodes, is_local_slice, args=[1,2]): peer=peer_index(i) if is_local_slice: sname=slice_name(i,ns) - slice=s[i].GetSlices(a[i],[sname])[0] + slice=s[i].GetSlices(a[i],{'name':[sname],'peer_id':None})[0] message='local' else: sname=slice_name(peer,ns) - slice=s[i].GetForeignSlices(a[i],[sname])[0] + slice=s[i].GetSlices(a[i],{'name':[sname],'~peer_id':None})[0] message='foreign' print '%02d: %s slice %s (e=%d) '%(i,message,sname,expected_nodes), slice_node_ids=slice['node_ids'] @@ -235,7 +245,7 @@ def check_slivers_1 (esn,args=[1,2]): def check_slivers_n (nn,esn,args=[1,2]): global plc,s,a for i in args: - nodename=node_name(i,i) + nodename=node_name(i,nn) ndict= s[i].GetSlivers(a[i],[nodename])[0] assert ndict['hostname'] == nodename slivers = ndict['slivers'] @@ -349,14 +359,17 @@ def test01_refresh (args=[1,2]): print 'got ',retcod #################### -def get_node_id(i,nodename): - return s[i].GetNodes(a[i],[nodename])[0]['node_id'] +# retrieves node_id from hostname - checks for local nodes only +def get_local_node_id(i,nodename): + return s[i].GetNodes(a[i],[nodename],None,'local')[0]['node_id'] +# clean all local nodes - foreign nodes are not supposed to be cleaned up manually def clean_all_nodes (args=[1,2]): global plc,s,a for i in args: print '%02d: Cleaning all nodes'%i - for node in s[i].GetNodes(a[i]): + loc_nodes = s[i].GetNodes(a[i],None,None,'local') + for node in loc_nodes: print '%02d: > Cleaning node %d'%(i,node['node_id']) s[i].DeleteNode(a[i],node['node_id']) @@ -369,7 +382,7 @@ def test02_node_n (nn,args=[1,2]): for i in args: nodename = node_name(i,nn) try: - get_node_id(i,nodename) + get_local_node_id(i,nodename) except: n=s[i].AddNode(a[i],1,{'hostname': nodename}) print '%02d: Added node %d %s'%(i,n,node_name(i,i)) @@ -382,7 +395,7 @@ def test02_delnode_n (nn,args=[1,2]): global plc,s,a for i in args: nodename = node_name(i,nn) - node_id = get_node_id (i,nodename) + node_id = get_local_node_id (i,nodename) retcod=s[i].DeleteNode(a[i],nodename) print '%02d: Deleted node %d, returns %s'%(i,node_id,retcod) @@ -391,14 +404,14 @@ def clean_all_slices (args=[1,2]): global plc,s,a for i in args: print '%02d: Cleaning all slices'%i - for slice in s[i].GetSlices(a[i]): + for slice in s[i].GetSlices(a[i],{'peer_id':None}): slice_id = slice['slice_id'] if slice_id not in system_slices_ids: print '%02d: > Cleaning slice %d'%(i,slice_id) s[i].DeleteSlice(a[i],slice_id) -def get_slice_id (i,name): - return s[i].GetSlices(a[i],[name])[0]['slice_id'] +def get_local_slice_id (i,name): + return s[i].GetSlices(a[i],{'name':[name],'peer_id':None})[0]['slice_id'] def test03_slice (args=[1,2]): for n in myrange(number_slices): @@ -434,7 +447,7 @@ def test04_node_slice_nl_n (nnl,ns,is_local, add_if_true, args=[1,2]): global plc,s,a for i in args: peer=peer_index(i) - slice_id = get_slice_id (i,slice_name (i,ns)) + slice_id = get_local_slice_id (i,slice_name (i,ns)) if is_local: hostnames=[node_name(i,nn) for nn in nnl] @@ -514,6 +527,14 @@ def test_all_nodes (): test01_refresh () check_nodes (number_nodes,number_nodes,) +def populate (): + test02_node() + test03_slice([1]) + test01_refresh ([1]) + test04_slice_add_lnode([1]) + test04_slice_add_fnode([1]) + test01_refresh() + def test_all_addslices (): # reset diff --git a/peers-test.mk b/peers-test.mk index a6c612b..3fc4f98 100644 --- a/peers-test.mk +++ b/peers-test.mk @@ -67,6 +67,10 @@ restart: @echo 'Restarting PLC' @chroot $(CHROOT) service plc restart +http: + @echo 'Restarting httpd' + @chroot $(CHROOT) /etc/plc.d/httpd stop ; chroot $(CHROOT) /etc/plc.d/httpd start + #################### UPGRADE=down up reconfig restart @@ -100,7 +104,7 @@ checkpoint: cp TestPeers.out TestPeers.ref ####### -HELP=rpm db-dump +HELP=rpm db-dump http help: @echo known targets: diff --git a/planetlab4.sql b/planetlab4.sql index 136e7e1..908459d 100644 --- a/planetlab4.sql +++ b/planetlab4.sql @@ -9,7 +9,7 @@ -- -- Copyright (C) 2006 The Trustees of Princeton University -- --- $Id: planetlab4.sql,v 1.40 2006/11/17 10:43:17 thierry Exp $ +-- $Id: planetlab4.sql,v 1.41 2006/11/17 19:31:08 tmack Exp $ -- -------------------------------------------------------------------------------- @@ -280,7 +280,11 @@ CREATE TABLE nodes ( -- Mandatory node_id serial PRIMARY KEY, -- Node identifier hostname text NOT NULL, -- Node hostname - site_id integer REFERENCES sites, -- At which site (clause NOT NULL removed for foreign_nodes) + -- temporarily removed NOT NULL clause for foreign_nodes + site_id integer REFERENCES sites, -- At which site + -- may be NULL for local_nodes + peer_id integer REFERENCES peers, -- From which peer + boot_state text REFERENCES boot_states NOT NULL DEFAULT 'inst', -- Node boot state deleted boolean NOT NULL DEFAULT false, -- Is deleted @@ -306,20 +310,11 @@ array_accum(node_id) AS node_ids FROM nodes GROUP BY site_id; --- Nodes - peers relationship -CREATE TABLE peer_node ( - peer_id integer REFERENCES peers NOT NULL, -- Peer identifier - node_id integer REFERENCES nodes NOT NULL, -- (Local) node identifier - PRIMARY KEY (peer_id, node_id), - UNIQUE (node_id) -- Nodes can only be at one peer -) WITH OIDS; -CREATE INDEX peer_node_peer_id_idx ON peer_node (peer_id); - -- Nodes at each peer CREATE VIEW peer_nodes AS SELECT peer_id, array_accum(node_id) AS node_ids -FROM peer_node +FROM nodes GROUP BY peer_id; -------------------------------------------------------------------------------- @@ -542,6 +537,8 @@ CREATE TABLE slices ( slice_id serial PRIMARY KEY, -- Slice identifier -- xxx temporarily remove the NOT NULL constraint site_id integer REFERENCES sites, -- Site identifier + peer_id integer REFERENCES peers, -- on which peer + name text NOT NULL, -- Slice name instantiation text REFERENCES slice_instantiations NOT NULL DEFAULT 'plc-instantiated', -- Slice state, e.g. plc-instantiated url text, -- Project URL @@ -594,19 +591,10 @@ FROM slices WHERE is_deleted is false GROUP BY site_id; --- Slices - peer relationship -CREATE TABLE peer_slice ( - peer_id integer REFERENCES peers NOT NULL, -- peer primary key - slice_id integer REFERENCES slices NOT NULL, -- node primary key - PRIMARY KEY (peer_id, slice_id) -) WITH OIDS; -CREATE INDEX peer_slice_peer_id_idx ON peer_slice (peer_id); -CREATE INDEX peer_slice_slice_id_idx ON peer_slice (slice_id); - CREATE VIEW peer_slices AS SELECT peer_id, array_accum(slice_id) AS slice_ids -FROM peer_slice +FROM slices GROUP BY peer_id; -- Slice membership @@ -817,11 +805,21 @@ LEFT JOIN person_sites USING (person_id) LEFT JOIN person_keys USING (person_id) LEFT JOIN person_slices USING (person_id); +CREATE VIEW view_peers AS +SELECT +peers.*, +peer_nodes.node_ids, +peer_slices.slice_ids +FROM peers +LEFT JOIN peer_nodes USING (peer_id) +LEFT JOIN peer_slices USING (peer_id); + CREATE VIEW view_nodes AS SELECT nodes.node_id, nodes.hostname, nodes.site_id, +nodes.peer_id, nodes.boot_state, nodes.deleted, nodes.model, @@ -839,40 +837,12 @@ COALESCE(node_pcus.ports, '{}') AS ports, COALESCE(node_conf_files.conf_file_ids, '{}') AS conf_file_ids, node_session.session_id AS session FROM nodes -LEFT JOIN peer_node USING (node_id) LEFT JOIN node_nodenetworks USING (node_id) LEFT JOIN node_nodegroups USING (node_id) LEFT JOIN node_slices USING (node_id) LEFT JOIN node_pcus USING (node_id) LEFT JOIN node_conf_files USING (node_id) -LEFT JOIN node_session USING (node_id) -WHERE peer_node.peer_id IS NULL; - -CREATE VIEW view_peers AS -SELECT -peers.*, -peer_nodes.node_ids, -peer_slices.slice_ids -FROM peers -LEFT JOIN peer_nodes USING (peer_id) -LEFT JOIN peer_slices USING (peer_id); - -CREATE VIEW view_foreign_nodes AS -SELECT -nodes.node_id, -nodes.hostname, -peer_node.peer_id, -nodes.boot_state, -nodes.model, -nodes.version, -CAST(date_part('epoch', nodes.date_created) AS bigint) AS date_created, -CAST(date_part('epoch', nodes.last_updated) AS bigint) AS last_updated, -COALESCE(node_slices.slice_ids, '{}') AS slice_ids, -nodes.deleted -FROM nodes -LEFT JOIN peer_node USING (node_id) -LEFT JOIN node_slices USING (node_id) -WHERE peer_node.peer_id IS NOT NULL; +LEFT JOIN node_session USING (node_id); CREATE VIEW view_nodegroups AS SELECT @@ -967,6 +937,7 @@ CREATE VIEW view_slices AS SELECT slices.slice_id, slices.site_id, +slices.peer_id, slices.name, slices.instantiation, slices.url, @@ -980,31 +951,9 @@ COALESCE(slice_nodes.node_ids, '{}') AS node_ids, COALESCE(slice_persons.person_ids, '{}') AS person_ids, COALESCE(slice_attributes.slice_attribute_ids, '{}') AS slice_attribute_ids FROM slices -LEFT JOIN peer_slice USING (slice_id) LEFT JOIN slice_nodes USING (slice_id) LEFT JOIN slice_persons USING (slice_id) -LEFT JOIN slice_attributes USING (slice_id) -WHERE peer_slice.peer_id IS NULL -AND slices.site_id IS NOT NULL -AND slices.creator_person_id IS NOT NULL; - -CREATE VIEW view_foreign_slices AS -SELECT -slices.slice_id, -slices.name, -peer_slice.peer_id, -slices.instantiation, -slices.url, -slices.description, -slices.max_nodes, -slices.is_deleted, -CAST(date_part('epoch', slices.created) AS bigint) AS created, -CAST(date_part('epoch', slices.expires) AS bigint) AS expires, -COALESCE(slice_nodes.node_ids, '{}') AS node_ids -FROM slices -LEFT JOIN peer_slice USING (slice_id) -LEFT JOIN slice_nodes USING (slice_id) -WHERE peer_slice.peer_id IS NOT NULL; +LEFT JOIN slice_attributes USING (slice_id); -- CREATE VIEW view_slice_attributes AS -- 2.43.0