From a1911156c062834ada3dd945c97e45563a980f56 Mon Sep 17 00:00:00 2001 From: Thierry Parmentelat Date: Mon, 13 Nov 2006 16:23:11 +0000 Subject: [PATCH] everything ready to receive foreign slices the actual part in RefreshPeer still needs to be written based on what's done with nodes also, peer_node and peer_slice now hold foreign_id, i.e. the id as known on the peer side, for being able to perform the right mapping at the time GetSlivers gets called --- PLC/ForeignSlices.py | 92 +++++++++++++++++++++++++++++++++ PLC/Methods/GetForeignSlices.py | 36 +++++++++++++ PLC/Methods/RefreshPeer.py | 40 +++++++------- PLC/Peers.py | 71 ++++++++++++++++--------- planetlab4.sql | 64 ++++++++++++++++++----- 5 files changed, 245 insertions(+), 58 deletions(-) create mode 100644 PLC/ForeignSlices.py create mode 100644 PLC/Methods/GetForeignSlices.py diff --git a/PLC/ForeignSlices.py b/PLC/ForeignSlices.py new file mode 100644 index 0000000..d5526eb --- /dev/null +++ b/PLC/ForeignSlices.py @@ -0,0 +1,92 @@ +# +# Thierry Parmentelat - INRIA +# +import time + +from types import StringTypes + +from PLC.Table import Row, Table +from PLC.Parameter import Parameter +from PLC.Filter import Filter + +class ForeignSlice (Row) : + """ + This object stores information about slices hosted on + other peering instances of myplc + """ + + table_name = 'slices' + primary_key = 'slice_id' + + fields = { + 'slice_id': Parameter (int, "Slice Id"), + 'name' : Parameter (str, "Slice name"), + 'peer_id': Parameter (str, "Peer id"), + 'instantiation': Parameter(str, "Slice instantiation state"), + 'url': Parameter(str, "URL further describing this slice", max = 254, nullok = True), + 'description': Parameter(str, "Slice description", max = 2048, nullok = True), + 'max_nodes': Parameter(int, "Maximum number of nodes that can be assigned to this slice"), + 'created': Parameter(int, "Date and time when slice was created, in seconds since UNIX epoch", ro = True), + 'expires': Parameter(int, "Date and time when slice expires, in seconds since UNIX epoch"), + } + + def __init__(self,api,fields={},uptodate=True): + Row.__init__(self,api,fields) + self.uptodate=uptodate + + def validate_created(self,timestamp): + return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(timestamp)) + + def validate_expires(self,timestamp): + return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(timestamp)) + + def purge_peer_slice (self,commit=True): + sql = "DELETE FROM peer_slice WHERE slice_id=%d"%self['slice_id'] + self.api.db.do(sql) + if commit: + self.api.db.commit() + + def delete (self, commit=True): + """ + Delete existing foreign slice. + """ + self.purge_peer_slice() + self['is_deleted']=True + self.sync(commit) + + +class ForeignSlices (Table): + def __init__ (self, api, foreign_slice_filter = None, columns = None): + Table.__init__(self, api, ForeignSlice, columns) + + sql = "" + sql += "SELECT %s FROM view_foreign_slices " % ", ".join(self.columns) + sql += "WHERE is_deleted IS False " + + if foreign_slice_filter is not None: + if isinstance(foreign_slice_filter, (list, tuple, set)): + # Separate the list into integers and strings + ints = filter(lambda x: isinstance(x, (int, long)), foreign_slice_filter) + strs = filter(lambda x: isinstance(x, StringTypes), foreign_slice_filter) + foreign_slice_filter = Filter(ForeignSlice.fields, {'slice_id': ints, 'name': strs}) + sql += " AND (%s)" % foreign_slice_filter.sql(api, "OR") + elif isinstance(foreign_slice_filter, dict): + foreign_slice_filter = Filter(ForeignSlice.fields, foreign_slice_filter) + sql += " AND (%s)" % foreign_slice_filter.sql(api, "AND") + + self.selectall(sql) + + # managing an index by slicename + def name_index(self): + if 'name' not in self.columns: + raise PLCFault,"ForeignSlices::name_index, name not selected" + self.index={} + for foreign_slice in self: + self.index[foreign_slice['name']]=foreign_slice + + def name_add_by(self,foreign_slice): + self.index[foreign_slice['name']]=foreign_slice + + def name_locate(self,name): + return self.index[name] + diff --git a/PLC/Methods/GetForeignSlices.py b/PLC/Methods/GetForeignSlices.py new file mode 100644 index 0000000..c1aa271 --- /dev/null +++ b/PLC/Methods/GetForeignSlices.py @@ -0,0 +1,36 @@ +# +# Thierry Parmentelat - INRIA +# + +from PLC.Faults import * +from PLC.Method import Method +from PLC.Parameter import Parameter, Mixed +from PLC.Filter import Filter +from PLC.Auth import Auth + +from PLC.ForeignSlices import ForeignSlice, ForeignSlices + +class GetForeignSlices(Method): + """ + Returns an array of structs containing details about foreign + slices. If foreign_slice_filter is specified and is an array of + foreign slice identifiers or names, or a struct of foreign slice + attributes, only foreign slices matching the filter will be + returned. If return_fields is specified, only the specified + details will be returned. + """ + + roles = ['admin'] + + accepts = [ + Auth(), + Mixed([Mixed(ForeignSlice.fields['slice_id'], + ForeignSlice.fields['name'])], + Filter(ForeignSlice.fields)), + Parameter([str], "List of fields to return", nullok = True) + ] + + returns = [ForeignSlice.fields] + + def call(self, auth, foreign_slice_filter = None, return_fields = None): + return ForeignSlices(self.api, foreign_slice_filter, return_fields) diff --git a/PLC/Methods/RefreshPeer.py b/PLC/Methods/RefreshPeer.py index 3657e3b..4a5abd0 100644 --- a/PLC/Methods/RefreshPeer.py +++ b/PLC/Methods/RefreshPeer.py @@ -11,7 +11,7 @@ from PLC.Auth import Auth from PLC.Peers import Peer, Peers from PLC.Persons import Person, Persons -from PLC.ForeignNodes import ForeignNode, ForeignNodes +##from PLC.ForeignNodes import ForeignNode, ForeignNodes class RefreshPeer(Method): @@ -19,7 +19,10 @@ class RefreshPeer(Method): Query a peer PLC for its list of nodes, and refreshes the local database accordingly - Returns the number of new nodes from that peer - may be negative + Returns a tuple containing + (*) the peer name + (*) the number of new nodes from that peer - may be negative + (*) the number of new slices from that peer - may be negative """ roles = ['admin'] @@ -33,16 +36,18 @@ class RefreshPeer(Method): ### retrieve peer info peers = Peers (self.api,[peer_id]) - if not peers: + try: + peer=peers[0] + except: raise PLCInvalidArgument,'no such peer_id:%d'%peer_id - peer=peers[0] ### retrieve account info person_id = peer['person_id'] persons = Persons (self.api,[person_id]) - if not persons: + try: + person = persons[0] + except: raise PLCInvalidArgument,'no such person_id:%d'%person_id - person = persons[0] ### build up foreign auth auth={ 'Username': person['email'], @@ -51,22 +56,15 @@ class RefreshPeer(Method): 'Role' : 'admin' } ## connect to the peer's API - url=peer['peer_url']+"/PLCAPI/" + url=peer['peer_url'] print 'url=',url apiserver = xmlrpclib.Server (url) print 'auth=',auth - current_peer_nodes = apiserver.GetNodes(auth) - print 'current_peer_nodes',current_peer_nodes - ## manual feed for tests - n11={'session': None, 'slice_ids': [], 'nodegroup_ids': [], 'last_updated': 1162884349, 'version': None, 'nodenetwork_ids': [], 'boot_state': 'inst', 'hostname': 'n11.plc1.org', 'site_id': 1, 'ports': None, 'pcu_ids': [], 'boot_nonce': None, 'node_id': 1, 'root_person_ids': [], 'key': None, 'date_created': 1162884349, 'model': None, 'conf_file_ids': [], 'ssh_rsa_key': None} - n12={'session': None, 'slice_ids': [], 'nodegroup_ids': [], 'last_updated': 1162884349, 'version': None, 'nodenetwork_ids': [], 'boot_state': 'inst', 'hostname': 'n12.plc1.org', 'site_id': 1, 'ports': None, 'pcu_ids': [], 'boot_nonce': None, 'node_id': 1, 'root_person_ids': [], 'key': None, 'date_created': 1162884349, 'model': None, 'conf_file_ids': [], 'ssh_rsa_key': None} - n21={'session': None, 'slice_ids': [], 'nodegroup_ids': [], 'last_updated': 1162884349, 'version': None, 'nodenetwork_ids': [], 'boot_state': 'boot', 'hostname': 'n21.plc2.org', 'site_id': 1, 'ports': None, 'pcu_ids': [], 'boot_nonce': None, 'node_id': 1, 'root_person_ids': [], 'key': None, 'date_created': 1162884349, 'model': None, 'conf_file_ids': [], 'ssh_rsa_key': None} - n22={'session': None, 'slice_ids': [], 'nodegroup_ids': [], 'last_updated': 1162884349, 'version': None, 'nodenetwork_ids': [], 'boot_state': 'boot', 'hostname': 'n22.plc2.org', 'site_id': 1, 'ports': None, 'pcu_ids': [], 'boot_nonce': None, 'node_id': 1, 'root_person_ids': [], 'key': None, 'date_created': 1162884349, 'model': None, 'conf_file_ids': [], 'ssh_rsa_key': None} - -# current_peer_nodes = [] -# print 'current_peer_nodes',current_peer_nodes - - nb_new_nodes = peer.refresh_nodes(current_peer_nodes) - - return nb_new_nodes + peer_get_nodes = apiserver.GetNodes(auth) + nb_new_nodes = peer.refresh_nodes(peer_get_nodes) + + peer_get_slices = apiserver.GetSlices(auth) + nb_new_slices = peer.refresh_slices(peer_get_slices) + + return (self.api.config.PLC_NAME,nb_new_nodes,nb_new_slices) diff --git a/PLC/Peers.py b/PLC/Peers.py index 872a867..e4598dc 100644 --- a/PLC/Peers.py +++ b/PLC/Peers.py @@ -3,7 +3,6 @@ # import re - from types import StringTypes from PLC.Faults import * @@ -38,9 +37,41 @@ class Peer (Row): raise invalid_url return url - def manage_node (self, foreign_node, add_if_true_del_if_false=True, commit=True): + def delete (self, commit=True): + """ + Delete peer + """ + + assert 'peer_id' in self + + # remove nodes depending on this peer + for foreign_node_id in self.get_foreign_nodes(): + try: + foreign_node = ForeignNodes(self.api,[foreign_node_id])[0] + foreign_node.delete(commit) + except: + print "Glitch : a foreign node instance was uncleanly deleted" + + # remove the peer + self['deleted'] = True + self.sync(commit) + + def get_foreign_nodes (self): """ - Add foreign node to a peer + returns a list of the foreign nodes in this peer + """ + sql="SELECT node_ids FROM peer_nodes WHERE peer_id=%d"%self['peer_id'] + node_ids = self.api.db.selectall(sql) + return node_ids[0]['node_ids'] + + def manage_node (self, foreign_node, foreign_id, commit=True): + """ + associate/dissociate a foreign node to/from a peer + foreign_node is a local object that describes a remote node + foreign_id is the unique id as provided by the remote peer + convention is: + if foreign_id is None : performs dissociation + otherwise: performs association """ assert 'peer_id' in self @@ -49,9 +80,9 @@ class Peer (Row): peer_id = self['peer_id'] node_id = foreign_node ['node_id'] - if add_if_true_del_if_false: + if foreign_id: ### ADDING - sql = "INSERT INTO peer_node VALUES (%d,%d)" % (peer_id,node_id) + sql = "INSERT INTO peer_node VALUES (%d,%d,%d)" % (peer_id,node_id,foreign_id) self.api.db.do(sql) if self['node_ids'] is None: self['node_ids']=[node_id,] @@ -65,7 +96,7 @@ class Peer (Row): if commit: self.api.db.commit() - def refresh_nodes (self, current_peer_nodes): + def refresh_nodes (self, peer_get_nodes): """ refreshes the foreign_nodes and peer_node tables expected input is the current list of nodes as returned by GetNodes @@ -92,10 +123,10 @@ class Peer (Row): ### xxx need to figure how to revert unix timestamp to db timestamp format remote_fields = ['boot_state','model','version','date_created','last_updated'] - ### scan the new entries, and mark them uptodate - for node in current_peer_nodes: + for node in peer_get_nodes: hostname = node['hostname'] + foreign_id = node ['node_id'] try: foreign_node = local_foreign_nodes.hostname_locate(hostname) if foreign_node['peer_id'] != peer_id: @@ -103,8 +134,10 @@ class Peer (Row): old_peer_id = foreign_node['peer_id'] old_peers=Peers(self.api,[peer_id]) assert old_peer[0] - old_peers[0].manage_node(foreign_node,False) - self.manage_node(foreign_node,True) + # remove from previous peer + old_peers[0].manage_node(foreign_node,None,False) + # add to new peer + self.manage_node(foreign_node,foreign_id,True) foreign_node['peer_id'] = peer_id ### update it anyway: copy other relevant fields for field in remote_fields: @@ -119,28 +152,20 @@ class Peer (Row): ### need to sync so we get a node_id new_foreign_node.sync() new_foreign_node.uptodate = True - self.manage_node(new_foreign_node,True) + self.manage_node(new_foreign_node,foreign_id,True) local_foreign_nodes.hostname_add_by(new_foreign_node) - ### delete entries that are not uptodate for foreign_node in local_foreign_nodes: if not foreign_node.uptodate: foreign_node.delete() - return len(current_peer_nodes)-old_count - + return len(peer_get_nodes)-old_count - def delete (self, commit=True): - """ - Delete peer - """ - - assert 'peer_id' in self - - self['deleted'] = True - self.sync(commit) + def refresh_slices (self, peer_get_slices): + return 0 + class Peers (Table): """ Maps to the peers table in the database diff --git a/planetlab4.sql b/planetlab4.sql index 9d24dc0..c68aecc 100644 --- a/planetlab4.sql +++ b/planetlab4.sql @@ -9,7 +9,7 @@ -- -- Copyright (C) 2006 The Trustees of Princeton University -- --- $Id: planetlab4.sql,v 1.30 2006/11/10 15:05:52 thierry Exp $ +-- $Id: planetlab4.sql,v 1.31 2006/11/10 17:00:21 thierry Exp $ -- -------------------------------------------------------------------------------- @@ -271,8 +271,6 @@ CREATE TABLE nodes ( hostname text NOT NULL, -- Node hostname site_id integer REFERENCES sites, -- At which site (clause NOT NULL removed for foreign_nodes) boot_state text REFERENCES boot_states NOT NULL DEFAULT 'inst', -- Node boot state --- cached boolean NOT NULL DEFAULT false, -- is this entry cached from a peer ? --- peer_id integer REFERENCES peers, -- if cached, then from what peer deleted boolean NOT NULL DEFAULT false, -- Is deleted -- Optional @@ -299,9 +297,10 @@ GROUP BY site_id; -- Nodes - peers relationship CREATE TABLE peer_node ( - peer_id integer REFERENCES peers NOT NULL, -- peer primary key - node_id integer REFERENCES nodes NOT NULL, -- node primary key - PRIMARY KEY (peer_id, node_id) + peer_id integer REFERENCES peers NOT NULL, -- peer primary key + node_id integer REFERENCES nodes NOT NULL, -- node primary key + foreign_id integer NOT NULL, + PRIMARY KEY (peer_id, node_id) ) WITH OIDS; CREATE INDEX peer_node_peer_id_idx ON peer_node (peer_id); CREATE INDEX peer_node_node_id_idx ON peer_node (node_id); @@ -313,13 +312,6 @@ array_accum(node_id) AS node_ids FROM peer_node GROUP BY peer_id; -CREATE VIEW view_peers AS -SELECT -peers.*, -peer_nodes.node_ids -FROM peers -LEFT JOIN peer_nodes USING (peer_id); - -------------------------------------------------------------------------------- -- Node groups -------------------------------------------------------------------------------- @@ -589,6 +581,22 @@ array_accum(slice_id) AS slice_ids FROM slices GROUP BY site_id; +-- Slices - peer relationship +CREATE TABLE peer_slice ( + peer_id integer REFERENCES peers NOT NULL, -- peer primary key + slice_id integer REFERENCES slices NOT NULL, -- node primary key + foreign_id integer NOT NULL, + PRIMARY KEY (peer_id, slice_id) +) WITH OIDS; +CREATE INDEX peer_slice_peer_id_idx ON peer_slice (peer_id); +CREATE INDEX peer_slice_slice_id_idx ON peer_slice (node_id); + +CREATE VIEW peer_slices AS +SELECT peer_id, +array_accum(slice_id) AS slice_ids +FROM peer_slice +GROUP BY peer_id; + -- Slice membership CREATE TABLE slice_person ( slice_id integer REFERENCES slices NOT NULL, -- Slice identifier @@ -825,6 +833,15 @@ LEFT JOIN node_conf_files USING (node_id) LEFT JOIN node_session USING (node_id) WHERE peer_node.peer_id IS NULL; +CREATE VIEW view_peers AS +SELECT +peers.*, +peer_nodes.node_ids, +peer_slices.slice_ids +FROM peers +LEFT JOIN peer_nodes USING (peer_id) +LEFT JOIN peer_slices USING (peer_id); + CREATE VIEW view_foreign_nodes AS SELECT nodes.node_id, @@ -948,10 +965,29 @@ COALESCE(slice_nodes.node_ids, '{}') AS node_ids, COALESCE(slice_persons.person_ids, '{}') AS person_ids, COALESCE(slice_attributes.slice_attribute_ids, '{}') AS slice_attribute_ids FROM slices +LEFT JOIN peer_slice USING (slice_id) LEFT JOIN slice_nodes USING (slice_id) LEFT JOIN slice_persons USING (slice_id) -LEFT JOIN slice_attributes USING (slice_id); +LEFT JOIN slice_attributes USING (slice_id) +WHERE peer_slice.peer_id IS NULL; + +CREATE VIEW view_foreign_slices AS +SELECT +slices.slice_id, +slices.name, +peer_slice.peer_id, +slices.instantiation, +slices.url, +slices.description, +slices.max_nodes, +slices.is_deleted, +CAST(date_part('epoch', slices.created) AS bigint) AS created, +CAST(date_part('epoch', slices.expires) AS bigint) AS expires +FROM slices +LEFT JOIN peer_slice USING (slice_id) +WHERE peer_slice.peer_id IS NOT NULL; +-- CREATE VIEW view_slice_attributes AS SELECT slice_attribute.slice_attribute_id, -- 2.45.2