From: Tony Mack Date: Tue, 25 Sep 2012 16:16:46 +0000 (-0400) Subject: Implement Slices X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=eb38c7f6dbea54b38bf2b9786d98d0ea95685320;p=plcapi.git Implement Slices --- diff --git a/PLC/Methods/UpdateSite.py b/PLC/Methods/UpdateSite.py index ea461a0d..936fdae7 100644 --- a/PLC/Methods/UpdateSite.py +++ b/PLC/Methods/UpdateSite.py @@ -4,12 +4,8 @@ from PLC.Parameter import Parameter, Mixed from PLC.Sites import Site, Sites from PLC.Auth import Auth -related_fields = Site.related_fields.keys() can_update = lambda (field, value): field in \ - ['name', 'abbreviated_name', 'login_base', - 'is_public', 'latitude', 'longitude', 'url', - 'max_slices', 'max_slivers', 'enabled', 'ext_consortium_id'] + \ - related_fields + ['name', 'enabled', 'description'] class UpdateSite(Method): """ @@ -24,56 +20,37 @@ class UpdateSite(Method): roles = ['admin', 'pi'] - site_fields = dict(filter(can_update, Site.fields.items() + Site.related_fields.items())) + site_fields = dict(filter(can_update, Site.fields.items())) accepts = [ Auth(), - Mixed(Site.fields['site_id'], - Site.fields['login_base']), + Site.fields['id'], site_fields ] returns = Parameter(int, '1 if successful') - def call(self, auth, site_id_or_login_base, site_fields): + def call(self, auth, site_id, site_fields): site_fields = dict(filter(can_update, site_fields.items())) # Get site information - sites = Sites(self.api, [site_id_or_login_base]) + sites = Sites(self.api, site_id) if not sites: raise PLCInvalidArgument, "No such site" site = sites[0] - if site['peer_id'] is not None: - raise PLCInvalidArgument, "Not a local site" - # Authenticated function assert self.caller is not None # If we are not an admin, make sure that the caller is a # member of the site. if 'admin' not in self.caller['roles']: - if site['site_id'] not in self.caller['site_ids']: + if site['id'] not in self.caller['tenant_id']: raise PLCPermissionDenied, "Not allowed to modify specified site" # Remove admin only fields - for key in 'max_slices', 'max_slivers', 'login_base': - if key in site_fields: - del site_fields[key] - - # Make requested associations - for field in related_fields: - if field in site_fields: - site.associate(auth, field, site_fields[field]) - site_fields.pop(field) site.update(site_fields) - site.update_last_updated(False) site.sync() - # Logging variables - self.event_objects = {'Site': [site['site_id']]} - self.message = 'Site %d updated: %s' % \ - (site['site_id'], ", ".join(site_fields.keys())) - return 1 diff --git a/PLC/NovaTable.py b/PLC/NovaTable.py index cb4518ff..ba198c23 100644 --- a/PLC/NovaTable.py +++ b/PLC/NovaTable.py @@ -1,6 +1,7 @@ +from PLC.Storage.Record import Record from PLC.Logger import logger -class NovaObject(dict): +class NovaObject(Record): fields = {} tags = {} def __init__(self, api, fields = {}, object=None): @@ -15,20 +16,6 @@ class NovaObject(dict): if hasattr(self.object, field): self[field] = getattr(self.object, field) - def todict(self): - return dict(self) - - def validate(self): - # Validate values before committing - for key, value in self.iteritems(): - if value is not None and hasattr(self, 'validate_' + key): - validate = getattr(self, 'validate_' + key) - self[key] = validate(value) - - def sync(self, insert=False, validate=True): - # Validate all specified fields - if validate: self.validate() - def __getattr__(self, name): return getattr(self.object, name) diff --git a/PLC/Parameter.py b/PLC/Parameter.py index 6268fcec..4b0cd08d 100644 --- a/PLC/Parameter.py +++ b/PLC/Parameter.py @@ -19,7 +19,8 @@ class Parameter: min = None, max = None, optional = None, ro = False, - nullok = False): + nullok = False, + primary_key = False): # Basic type of the parameter. Must be a builtin type # that can be marshalled by XML-RPC. self.type = typeval @@ -43,6 +44,8 @@ class Parameter: # Whether the DB field can be NULL. self.nullok = nullok + self.primary_key = primary_key + def type(self): return self.type diff --git a/PLC/Slices.py b/PLC/Slices.py index db0cd050..e3a4c13a 100644 --- a/PLC/Slices.py +++ b/PLC/Slices.py @@ -2,18 +2,23 @@ from types import StringTypes import time import re +from sqlalchemy import Column, Integer, String, DateTime +from sqlalchemy import Table, Column, MetaData, join, ForeignKey +from sqlalchemy.orm import relationship, backref +from sqlalchemy.orm import column_property +from sqlalchemy.orm import object_mapper +from sqlalchemy.orm import validates + +from PLC.Storage.AlchemyObject import AlchemyObj from PLC.Faults import * from PLC.Parameter import Parameter, Mixed from PLC.Filter import Filter from PLC.Debug import profile -from PLC.Table import Row, Table -from PLC.SliceInstantiations import SliceInstantiation, SliceInstantiations from PLC.Nodes import Node from PLC.Persons import Person, Persons -from PLC.SliceTags import SliceTag from PLC.Timestamp import Timestamp -class Slice(Row): +class Slice(AlchemyObj): """ Representation of a row in the slices table. To use, optionally instantiate with a dict of values. Update as you would a @@ -21,14 +26,12 @@ class Slice(Row): with a dict of values. """ - table_name = 'slices' - primary_key = 'slice_id' - join_tables = ['slice_node', 'slice_person', 'slice_tag', 'peer_slice', 'node_slice_whitelist', 'leases', ] + tablename = 'slices' + fields = { - 'slice_id': Parameter(int, "Slice identifier"), + 'id': Parameter(int, "Slice identifier", primary_key=True), 'site_id': Parameter(int, "Identifier of the site to which this slice belongs"), 'name': Parameter(str, "Slice name", max = 32), - 'instantiation': Parameter(str, "Slice instantiation state"), 'url': Parameter(str, "URL further describing this slice", max = 254, nullok = True), 'description': Parameter(str, "Slice description", max = 2048, nullok = True), 'max_nodes': Parameter(int, "Maximum number of nodes that can be assigned to this slice"), @@ -37,18 +40,7 @@ class Slice(Row): 'expires': Parameter(int, "Date and time when slice expires, in seconds since UNIX epoch"), 'node_ids': Parameter([int], "List of nodes in this slice", ro = True), 'person_ids': Parameter([int], "List of accounts that can use this slice", ro = True), - 'slice_tag_ids': Parameter([int], "List of slice attributes", ro = True), - 'peer_id': Parameter(int, "Peer to which this slice belongs", nullok = True), - 'peer_slice_id': Parameter(int, "Foreign slice identifier at peer", nullok = True), } - related_fields = { - 'persons': [Mixed(Parameter(int, "Person identifier"), - Parameter(str, "Email address"))], - 'nodes': [Mixed(Parameter(int, "Node identifier"), - Parameter(str, "Fully qualified hostname"))] - } - - view_tags_name="view_slice_tags" tags = {} def validate_name(self, name): @@ -67,168 +59,37 @@ class Slice(Row): conflicts = Slices(self.api, [name]) for slice in conflicts: - if 'slice_id' not in self or self['slice_id'] != slice['slice_id']: + if 'id' not in self or self['id'] != slice.id: raise PLCInvalidArgument, "Slice name already in use, %s"%name return name - def validate_instantiation(self, instantiation): - instantiations = [row['instantiation'] for row in SliceInstantiations(self.api)] - if instantiation not in instantiations: - raise PLCInvalidArgument, "No such instantiation state" - - return instantiation - - validate_created = Row.validate_timestamp - def validate_expires(self, expires): # N.B.: Responsibility of the caller to ensure that expires is # not too far into the future. check_future = not ('is_deleted' in self and self['is_deleted']) return Timestamp.sql_validate( expires, check_future = check_future) - add_person = Row.add_object(Person, 'slice_person') - remove_person = Row.remove_object(Person, 'slice_person') - - add_node = Row.add_object(Node, 'slice_node') - remove_node = Row.remove_object(Node, 'slice_node') - - add_to_node_whitelist = Row.add_object(Node, 'node_slice_whitelist') - delete_from_node_whitelist = Row.remove_object(Node, 'node_slice_whitelist') - - def associate_persons(self, auth, field, value): - """ - Adds persons found in value list to this slice (using AddPersonToSlice). - Deletes persons not found in value list from this slice (using DeletePersonFromSlice). - """ - - assert 'person_ids' in self - assert 'slice_id' in self - assert isinstance(value, list) - - (person_ids, emails) = self.separate_types(value)[0:2] - - # Translate emails into person_ids - if emails: - persons = Persons(self.api, emails, ['person_id']).dict('person_id') - person_ids += persons.keys() - - # Add new ids, remove stale ids - if self['person_ids'] != person_ids: - from PLC.Methods.AddPersonToSlice import AddPersonToSlice - from PLC.Methods.DeletePersonFromSlice import DeletePersonFromSlice - new_persons = set(person_ids).difference(self['person_ids']) - stale_persons = set(self['person_ids']).difference(person_ids) - - for new_person in new_persons: - AddPersonToSlice.__call__(AddPersonToSlice(self.api), auth, new_person, self['slice_id']) - for stale_person in stale_persons: - DeletePersonFromSlice.__call__(DeletePersonFromSlice(self.api), auth, stale_person, self['slice_id']) - - def associate_nodes(self, auth, field, value): - """ - Adds nodes found in value list to this slice (using AddSliceToNodes). - Deletes nodes not found in value list from this slice (using DeleteSliceFromNodes). - """ - - from PLC.Nodes import Nodes - - assert 'node_ids' in self - assert 'slice_id' in self - assert isinstance(value, list) - - (node_ids, hostnames) = self.separate_types(value)[0:2] - - # Translate hostnames into node_ids - if hostnames: - nodes = Nodes(self.api, hostnames, ['node_id']).dict('node_id') - node_ids += nodes.keys() - - # Add new ids, remove stale ids - if self['node_ids'] != node_ids: - from PLC.Methods.AddSliceToNodes import AddSliceToNodes - from PLC.Methods.DeleteSliceFromNodes import DeleteSliceFromNodes - new_nodes = set(node_ids).difference(self['node_ids']) - stale_nodes = set(self['node_ids']).difference(node_ids) - - if new_nodes: - AddSliceToNodes.__call__(AddSliceToNodes(self.api), auth, self['slice_id'], list(new_nodes)) - if stale_nodes: - DeleteSliceFromNodes.__call__(DeleteSliceFromNodes(self.api), auth, self['slice_id'], list(stale_nodes)) - def associate_slice_tags(self, auth, fields, value): - """ - Deletes slice_tag_ids not found in value list (using DeleteSliceTag). - Adds slice_tags if slice_fields w/o slice_id is found (using AddSliceTag). - Updates slice_tag if slice_fields w/ slice_id is found (using UpdateSlceiAttribute). - """ - - assert 'slice_tag_ids' in self - assert isinstance(value, list) - - (attribute_ids, blank, attributes) = self.separate_types(value) - - # There is no way to add attributes by id. They are - # associated with a slice when they are created. - # So we are only looking to delete here - if self['slice_tag_ids'] != attribute_ids: - from PLC.Methods.DeleteSliceTag import DeleteSliceTag - stale_attributes = set(self['slice_tag_ids']).difference(attribute_ids) - - for stale_attribute in stale_attributes: - DeleteSliceTag.__call__(DeleteSliceTag(self.api), auth, stale_attribute['slice_tag_id']) - - # If dictionary exists, we are either adding new - # attributes or updating existing ones. - if attributes: - from PLC.Methods.AddSliceTag import AddSliceTag - from PLC.Methods.UpdateSliceTag import UpdateSliceTag + #add_person = Row.add_object(Person, 'slice_person') + #remove_person = Row.remove_object(Person, 'slice_person') - added_attributes = filter(lambda x: 'slice_tag_id' not in x, attributes) - updated_attributes = filter(lambda x: 'slice_tag_id' in x, attributes) + #add_node = Row.add_object(Node, 'slice_node') + #remove_node = Row.remove_object(Node, 'slice_node') - for added_attribute in added_attributes: - if 'tag_type' in added_attribute: - type = added_attribute['tag_type'] - elif 'tag_type_id' in added_attribute: - type = added_attribute['tag_type_id'] - else: - raise PLCInvalidArgument, "Must specify tag_type or tag_type_id" + #add_to_node_whitelist = Row.add_object(Node, 'node_slice_whitelist') + #delete_from_node_whitelist = Row.remove_object(Node, 'node_slice_whitelist') - if 'value' in added_attribute: - value = added_attribute['value'] - else: - raise PLCInvalidArgument, "Must specify a value" - - if 'node_id' in added_attribute: - node_id = added_attribute['node_id'] - else: - node_id = None - - if 'nodegroup_id' in added_attribute: - nodegroup_id = added_attribute['nodegroup_id'] - else: - nodegroup_id = None - - AddSliceTag.__call__(AddSliceTag(self.api), auth, self['slice_id'], type, value, node_id, nodegroup_id) - for updated_attribute in updated_attributes: - attribute_id = updated_attribute.pop('slice_tag_id') - if attribute_id not in self['slice_tag_ids']: - raise PLCInvalidArgument, "Attribute doesnt belong to this slice" - else: - UpdateSliceTag.__call__(UpdateSliceTag(self.api), auth, attribute_id, updated_attribute) - - def sync(self, commit = True): + def sync(self, commit = True, validate=True): """ Add or update a slice. """ - - # Before a new slice is added, delete expired slices - if 'slice_id' not in self: + AlchemyObj.sync(self, commit, validate) + if 'id' not in self: + # Before a new slice is added, delete expired slices expired = Slices(self.api, expires = -int(time.time())) for slice in expired: slice.delete(commit) - - Row.sync(self, commit) + Slice().insert(dict(self)) def delete(self, commit = True): """ @@ -247,49 +108,26 @@ class Slice(Row): self.sync(commit) -class Slices(Table): +class Slices(list): """ Representation of row(s) from the slices table in the database. """ def __init__(self, api, slice_filter = None, columns = None, expires = int(time.time())): - Table.__init__(self, api, Slice, columns) - + # the view that we're selecting upon: start with view_slices - view = "view_slices" - # as many left joins as requested tags - for tagname in self.tag_columns: - view= "%s left join %s using (%s)"%(view,Slice.tagvalue_view_name(tagname), - Slice.primary_key) - - sql = "SELECT %s FROM %s WHERE is_deleted IS False" % \ - (", ".join(self.columns.keys()+self.tag_columns.keys()),view) - - if expires is not None: - if expires >= 0: - sql += " AND expires > %d" % expires - else: - expires = -expires - sql += " AND expires < %d" % expires - - if slice_filter is not None: - if isinstance(slice_filter, (list, tuple, set)): - # Separate the list into integers and strings - ints = filter(lambda x: isinstance(x, (int, long)), slice_filter) - strs = filter(lambda x: isinstance(x, StringTypes), slice_filter) - slice_filter = Filter(Slice.fields, {'slice_id': ints, 'name': strs}) - sql += " AND (%s) %s" % slice_filter.sql(api, "OR") - elif isinstance(slice_filter, dict): - slice_filter = Filter(Slice.fields, slice_filter) - sql += " AND (%s) %s" % slice_filter.sql(api, "AND") - elif isinstance (slice_filter, StringTypes): - slice_filter = Filter(Slice.fields, {'name':slice_filter}) - sql += " AND (%s) %s" % slice_filter.sql(api, "AND") - elif isinstance (slice_filter, (int, long)): - slice_filter = Filter(Slice.fields, {'slice_id':slice_filter}) - sql += " AND (%s) %s" % slice_filter.sql(api, "AND") - else: - raise PLCInvalidArgument, "Wrong slice filter %r"%slice_filter - - self.selectall(sql) + if not slice_filter: + slices = Slice().select() + elif isinstance (slice_filter, StringTypes): + slices = Slice().select(filter={'name': slice_filter}) + elif isinstance(slice_filter, dict): + slices = Slice().select(filter=slice_filter) + elif isinstance(slice_filter, (list, tuple, set)): + slices = Slice().select() + slices = [slice for slice in slices if slice.id in slice_filter] + else: + raise PLCInvalidArgument, "Wrong slice filter %r"%slice_filter + + for slice in slices: + self.append(slice) diff --git a/PLC/Storage/AlchemyObject.py b/PLC/Storage/AlchemyObject.py new file mode 100644 index 00000000..6186948d --- /dev/null +++ b/PLC/Storage/AlchemyObject.py @@ -0,0 +1,77 @@ +from datetime import datetime + +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Table, Column, MetaData, join, ForeignKey +from sqlalchemy import Column, Integer, String +from sqlalchemy import Table, Column, MetaData, join, ForeignKey +from sqlalchemy.orm import relationship, backref +from sqlalchemy.orm import column_property +from sqlalchemy.orm import mapper, object_mapper +from sqlalchemy.orm import validates +from sqlalchemy import ForeignKey + +from PLC.Storage.Alchemy import Base, engine, dbsession +from PLC.Storage.Record import Record +from PLC.Timestamp import Timestamp +from PLC.Logger import logger + +class AlchemyObj(Record): + + def __init__(self, api=None, fields = {}, object=None): + self.api=api + Record.__init__(self, dict=fields) + + def __iter__(self): + self._i = iter(object_mapper(self).columns) + return self + + def next(self): + n = self._i.next().name + return n, getattr(self, n) + + def get_table(self): + metadata = MetaData(engine) + table = Table(self.tablename, metadata) + for field in self.fields: + param = self.fields[field] + type = String + if param.type == int: + type = Integer + elif param.type == datetime: + type = DateTime + column = Column(field, type, primary_key=param.primary_key) + table.append_column(column) + if not table.exists(): + table.create() + return table + + def create(self): + table = self.get_table() + table.create() + + def insert(self, values): + table = self.get_table() + result = dbsession.execute(table.insert().values(values)) + dbsession.commit() + return result + + + def updatedb(self, values): + table = self.get_table() + result = dbsession.execute(table.update(), values) + dbsession.commit() + return result + + def delete(self, filter={}): + class Cls(object): pass + table = self.get_table() + clsmapper = mapper(Cls, table) + result = dbsession.query(clsmapper).filter_by(*filter).delete() + dbsession.commit() + return result + + def select(self, filter={}): + class Cls(object): pass + table = self.get_table() + clsmapper = mapper(Cls, table) + return dbsession.query(clsmapper).filter_by(**filter) diff --git a/PLC/Timestamp.py b/PLC/Timestamp.py index 9f382ece..2b01a438 100644 --- a/PLC/Timestamp.py +++ b/PLC/Timestamp.py @@ -123,7 +123,37 @@ class Timestamp: return result else: raise PLCInvalidArgument, "Timestamp %r - unsupported type %r"%(input,type(input)) + + def utcparse(input): + """ Translate a string into a time using dateutil.parser.parse but make + sure it's in UTC time and strip the timezone, so that it's compatible + with normal datetime.datetime objects. + + For safety this can also handle inputs that are either timestamps, or + datetimes + """ + # prepare the input for the checks below by + # casting strings ('1327098335') to ints + if isinstance(input, StringTypes): + try: + input = int(input) + except ValueError: + pass + + if isinstance (input, datetime.datetime): + return input + elif isinstance (input, StringTypes): + t = dateutil.parser.parse(input) + if t.utcoffset() is not None: + t = t.utcoffset() + t.replace(tzinfo=None) + return t + elif isinstance (input, (int,float,long)): + return datetime.datetime.fromtimestamp(input) + else: + raise + def string(input): + return datetime.datetime.strftime(Timestamp.utcparse(input), Timestamp.sql_format) # utility for displaying durations # be consistent in avoiding the datetime stuff diff --git a/config/default_config b/config/default_config index d3f87f51..ecd12d62 100644 --- a/config/default_config +++ b/config/default_config @@ -17,4 +17,10 @@ admin_password=admin admin_tenant=admin url=http://localhost:5000/v2.0/ +[db] +user=plcuser +password=plcuser +name=planetlab +host=locahost +port=5432