From: jcnelson Date: Thu, 10 Jul 2014 23:33:34 +0000 (-0400) Subject: Updated syndicate admin and models: X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=575a1353cfb6fd98960278e2146ad6d1c4e49be4;p=plstackapi.git Updated syndicate admin and models: * Added slice-specific secret, autogenerated when a Volume is mounted on a slice for the first time * Rename syndicate to syndicate_storage to avoid package import conflicts * Use more sensible field names * Remove BitField usage --- diff --git a/planetstack/syndicate/__init__.py b/planetstack/syndicate/__init__.py deleted file mode 100644 index 2b86d97..0000000 --- a/planetstack/syndicate/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from syndicate.models import * diff --git a/planetstack/syndicate/models.py b/planetstack/syndicate/models.py deleted file mode 100644 index 656e881..0000000 --- a/planetstack/syndicate/models.py +++ /dev/null @@ -1,93 +0,0 @@ -from core.models import User,Site,Service,SingletonModel,PlCoreBase,Slice -import os -from django.db import models -from django.forms.models import model_to_dict -from bitfield import BitField -from django.core.exceptions import ValidationError - -# Create your models here. - -class SyndicateService(SingletonModel,Service): - class Meta: - app_label = "syndicate" - verbose_name = "Syndicate Service" - verbose_name_plural = "Syndicate Service" - - def __unicode__(self): return u'Syndicate Service' - - -class SyndicatePrincipal(PlCoreBase): - class Meta: - app_label = "syndicate" - - # for now, this is a user email address - principal_id = models.TextField() - public_key_pem = models.TextField() - sealed_private_key = models.TextField() - - def __unicode__self(self): return "%s" % self.principal_id - - -class Volume(PlCoreBase): - class Meta: - app_label = "syndicate" - - name = models.CharField(max_length=64, help_text="Human-readable, searchable name of the Volume") - - owner_id = models.ForeignKey(User, verbose_name='Owner') - - description = models.TextField(null=True, blank=True,max_length=130, help_text="Human-readable description of what this Volume is used for.") - blocksize = models.PositiveIntegerField(help_text="Number of bytes per block.") - private = models.BooleanField(default=True, help_text="Indicates if the Volume is visible to users other than the Volume Owner and Syndicate Administrators.") - archive = models.BooleanField(default=False, help_text="Indicates if this Volume is read-only, and only an Aquisition Gateway owned by the Volume owner (or Syndicate admin) can write to it.") - - CAP_READ_DATA = 1 - CAP_WRITE_DATA = 2 - CAP_HOST_DATA = 4 - - # NOTE: preserve order of capabilities here... - default_gateway_caps = BitField(flags=("read data", "write data", "host files"), verbose_name='Default User Capabilities') - - def __unicode__(self): return self.name - - -class VolumeAccessRight(PlCoreBase): - class Meta: - app_label = "syndicate" - - owner_id = models.ForeignKey(User, verbose_name='user') - - volume = models.ForeignKey(Volume) - gateway_caps = BitField(flags=("read data", "write data", "host files"), verbose_name="User Capabilities") - - def __unicode__(self): return "%s-%s" % (self.owner_id.email, self.volume.name) - - -class VolumeSlice(PlCoreBase): - class Meta: - app_label = "syndicate" - - volume_id = models.ForeignKey(Volume, verbose_name="Volume") - slice_id = models.ForeignKey(Slice, verbose_name="Slice") - gateway_caps = BitField(flags=("read data", "write data", "host files"), verbose_name="Slice Capabilities") - - peer_portnum = models.PositiveIntegerField(help_text="User Gateway port", verbose_name="Client peer-to-peer cache port") - replicate_portnum = models.PositiveIntegerField(help_text="Replica Gateway port", verbose_name="Replication service port") - - credentials_blob = models.TextField(null=True, blank=True, help_text="Encrypted slice credentials") - - def __unicode__(self): return "%s-%s" % (self.volume_id.name, self.slice_id.name) - - def clean(self): - """ - Verify that our fields are in order: - * peer_portnum and replicate_portnum have to be valid port numbers between 1025 and 65534 - * peer_portnum and replicate_portnum cannot be changed once set. - """ - - if self.peer_portnum < 1025 or self.peer_portnum > 65534: - raise ValidationError( "Client peer-to-peer cache port number must be between 1025 and 65534" ) - - if self.replicate_portnum < 1025 or self.replicate_portnum > 65534: - raise ValidationError( "Replication service port number must be between 1025 and 65534" ) - diff --git a/planetstack/syndicate_storage/__init__.py b/planetstack/syndicate_storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/planetstack/syndicate/admin.py b/planetstack/syndicate_storage/admin.py similarity index 66% rename from planetstack/syndicate/admin.py rename to planetstack/syndicate_storage/admin.py index e9f499c..418ba2b 100644 --- a/planetstack/syndicate/admin.py +++ b/planetstack/syndicate_storage/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -from syndicate.models import * +from syndicate_storage.models import * from django import forms from django.utils.safestring import mark_safe from django.contrib.auth.admin import UserAdmin @@ -12,14 +12,12 @@ from django.contrib.contenttypes import generic from suit.widgets import LinkedSelect from core.admin import ReadOnlyTabularInline,ReadOnlyAwareAdmin,SingletonAdmin,SliceInline,ServiceAttrAsTabInline,PlanetStackBaseAdmin, PlStackTabularInline,SliceROInline,ServiceAttrAsTabROInline from suit.widgets import LinkedSelect -from bitfield import BitField -from bitfield.forms import BitFieldCheckboxSelectMultiple from django.core.exceptions import ValidationError, ObjectDoesNotExist class SyndicateServiceAdmin(SingletonAdmin,ReadOnlyAwareAdmin): model = SyndicateService - verbose_name = "Syndicate Service" - verbose_name_plural = "Syndicate Service" + verbose_name = "Syndicate Storage" + verbose_name_plural = "Syndicate Storage" list_display = ("name","enabled") fieldsets = [(None, {'fields': ['name','enabled','versionNumber', 'description',], 'classes':['suit-tab suit-tab-general']})] inlines = [SliceInline,ServiceAttrAsTabInline] @@ -27,7 +25,7 @@ class SyndicateServiceAdmin(SingletonAdmin,ReadOnlyAwareAdmin): user_readonly_fields = ['name','enabled','versionNumber','description'] user_readonly_inlines = [SliceROInline, ServiceAttrAsTabROInline] - suit_form_tabs =(('general', 'Syndicate Service Details'), + suit_form_tabs =(('general', 'Syndicate Storage Details'), ('slices','Slices'), ('serviceattrs','Additional Attributes'), ) @@ -37,34 +35,18 @@ class VolumeAccessRightForUserROInline(ReadOnlyTabularInline): model = VolumeAccessRight extra = 0 suit_classes = 'suit-tab suit-tab-volumeAccessRights' - fields = ['volume','gateway_caps'] + fields = ['volume','cap_read_data', 'cap_write_data', 'cap_host_data'] class VolumeAccessRightROInline(ReadOnlyTabularInline): model = VolumeAccessRight extra = 0 suit_classes = 'suit-tab suit-tab-volumeAccessRights' - fields = ['owner_id','gateway_caps'] + fields = ['owner_id','cap_read_data', 'cap_write_data', 'cap_host_data'] class VolumeAccessRightInline(PlStackTabularInline): model = VolumeAccessRight extra = 0 suit_classes = 'suit-tab suit-tab-volumeAccessRights' - formfield_overrides = { - BitField: {'widget': BitFieldCheckboxSelectMultiple} - } - fields = ('owner_id', 'gateway_caps') - -class VolumeInline(PlStackTabularInline): - model = Volume - extra = 0 - suit_classes = 'suit-tab suit-tab-volumes' - fields = ['name', 'owner_id'] - -class VolumeROInline(ReadOnlyTabularInline): - model = Volume - extra = 0 - suit_classes = 'suit-tab suit-tab-volumes' - fields = ['name', 'owner_id'] class VolumeSliceFormSet( forms.models.BaseInlineFormSet ): @@ -94,29 +76,29 @@ class VolumeSliceFormSet( forms.models.BaseInlineFormSet ): cleaned_data = form.cleaned_data except AttributeError: continue - + # verify that the ports haven't changed volume_pk = cleaned_data['volume_id'].pk slice_pk = cleaned_data['slice_id'].pk - if not cleaned_data.has_key('peer_portnum'): - raise ValidationError("Missing client peer-to-peer cache port number") + if not cleaned_data.has_key('UG_portnum'): + raise ValidationError("Missing UG port number") - if not cleaned_data.has_key('replicate_portnum'): - raise ValidationError("Missing replication service port number") + if not cleaned_data.has_key('RG_portnum'): + raise ValidationError("Missing RG port number") - rc1, old_peer_port = VolumeSliceFormSet.verify_unchanged( volume_pk, slice_pk, 'peer_portnum', cleaned_data['peer_portnum'] ) - rc2, old_replicate_port = VolumeSliceFormSet.verify_unchanged( volume_pk, slice_pk, 'replicate_portnum', cleaned_data['replicate_portnum'] ) + rc1, old_peer_port = VolumeSliceFormSet.verify_unchanged( volume_pk, slice_pk, 'UG_portnum', cleaned_data['UG_portnum'] ) + rc2, old_replicate_port = VolumeSliceFormSet.verify_unchanged( volume_pk, slice_pk, 'RG_portnum', cleaned_data['RG_portnum'] ) err1str = "" err2str = "" if not rc1: - err1str = "change %s back to %s" % (cleaned_data['peer_portnum'], old_peer_port) + err1str = "change %s back to %s" % (cleaned_data['UG_portnum'], old_peer_port) if not rc2: - err2str = " and change %s back to %s" % (cleaned_data['replicate_portnum'], old_replicate_port ) + err2str = " and change %s back to %s" % (cleaned_data['RG_portnum'], old_replicate_port ) if not rc1 or not rc2: - raise ValidationError("Port numbers cannot be changed once they are set. Please %s %s" % (err1str, err2str)) + raise ValidationError("At this time, port numbers cannot be changed once they are set. Please %s %s" % (err1str, err2str)) @@ -124,8 +106,7 @@ class VolumeSliceInline(PlStackTabularInline): model = VolumeSlice extra = 0 suit_classes = 'suit-tab suit-tab-volumeSlices' - fields = ['volume_id', 'slice_id', 'gateway_caps', 'peer_portnum', 'replicate_portnum'] - formfield_overrides = { BitField: {'widget': BitFieldCheckboxSelectMultiple},} + fields = ['volume_id', 'slice_id', 'cap_read_data', 'cap_write_data', 'cap_host_data', 'UG_portnum', 'RG_portnum'] formset = VolumeSliceFormSet @@ -136,19 +117,18 @@ class VolumeSliceROInline(ReadOnlyTabularInline): model = VolumeSlice extra = 0 suit_classes = 'suit-tab suit-tab-volumeSlices' - fields = ['volume_id', 'slice_id', 'gateway_caps', 'peer_portnum', 'replicate_portnum'] - formfield_overrides = { BitField: {'widget': BitFieldCheckboxSelectMultiple},} + fields = ['volume_id', 'slice_id', 'cap_read_data', 'cap_write_data', 'cap_host_data', 'UG_portnum', 'RG_portnum'] formset = VolumeSliceFormSet readonly_fields = ['credentials_blob'] - + class VolumeAdmin(ReadOnlyAwareAdmin): model = Volume def get_readonly_fields(self, request, obj=None ): - always_readonly = list(super(VolumeAdmin, self).get_readonly_fields(request, obj)) + always_readonly = [] if obj == None: # all fields are editable on add return always_readonly @@ -160,29 +140,27 @@ class VolumeAdmin(ReadOnlyAwareAdmin): list_display = ['name', 'owner_id'] - formfield_overrides = { BitField: {'widget': BitFieldCheckboxSelectMultiple},} + detailsFieldList = ['name', 'owner_id', 'description','blocksize', 'private','archive', 'cap_read_data', 'cap_write_data', 'cap_host_data' ] - #detailsFieldList = ['name', 'owner_id', 'description','file_quota','blocksize', 'private','archive', 'default_gateway_caps' ] - detailsFieldList = ['name', 'owner_id', 'description','blocksize', 'private','archive', 'default_gateway_caps' ] - fieldsets = [ (None, {'fields': detailsFieldList, 'classes':['suit-tab suit-tab-general']}), - #(None, {'fields': keyList, 'classes':['suit-tab suit-tab-volumeKeys']}), ] inlines = [VolumeAccessRightInline, VolumeSliceInline] - user_readonly_fields = ['name','owner_id','description','blocksize','private','default_gateway_caps'] + user_readonly_fields = ['name','owner_id','description','blocksize','private', 'archive', 'cap_read_data', 'cap_write_data', 'cap_host_data'] user_readonly_inlines = [VolumeAccessRightROInline, VolumeSliceROInline] suit_form_tabs =(('general', 'Volume Details'), - #('volumeKeys', 'Access Keys'), ('volumeSlices', 'Slices'), - ('volumeAccessRights', 'Volume Access Rights'), - ) + ('volumeAccessRights', 'Volume Access Rights')) + + def queryset(self, request): + # only show volumes that are public, or owned by the caller + return Volume.select_by_user(request.user) + - # left panel: admin.site.register(SyndicateService, SyndicateServiceAdmin) admin.site.register(Volume, VolumeAdmin) diff --git a/planetstack/syndicate_storage/models.py b/planetstack/syndicate_storage/models.py new file mode 100644 index 0000000..a7e0c90 --- /dev/null +++ b/planetstack/syndicate_storage/models.py @@ -0,0 +1,259 @@ +from core.models import User,Site,Service,SingletonModel,PlCoreBase,Slice,SlicePrivilege +import os +from django.db import models +from django.db.models import Q +from django.forms.models import model_to_dict +from django.core.exceptions import ValidationError, ObjectDoesNotExist + +# Create your models here. + +class SyndicateService(SingletonModel,Service): + class Meta: + app_label = "syndicate_storage" + verbose_name = "Syndicate Service" + verbose_name_plural = "Syndicate Service" + + def __unicode__(self): return u'Syndicate Service' + + +class SyndicatePrincipal(PlCoreBase): + class Meta: + app_label = "syndicate_storage" + + # for now, this is a user email address + principal_id = models.TextField() + public_key_pem = models.TextField() + sealed_private_key = models.TextField() + + def __unicode__self(self): return "%s" % self.principal_id + + +class Volume(PlCoreBase): + class Meta: + app_label = "syndicate_storage" + + name = models.CharField(max_length=64, help_text="Human-readable, searchable name of the Volume") + + owner_id = models.ForeignKey(User, verbose_name='Owner') + + description = models.TextField(null=True, blank=True,max_length=130, help_text="Human-readable description of what this Volume is used for.") + blocksize = models.PositiveIntegerField(help_text="Number of bytes per block.") + private = models.BooleanField(default=True, help_text="Indicates if the Volume is visible to users other than the Volume Owner and Syndicate Administrators.") + archive = models.BooleanField(default=False, help_text="Indicates if this Volume is read-only, and only an Aquisition Gateway owned by the Volume owner (or Syndicate admin) can write to it.") + + cap_read_data = models.BooleanField(default=True, help_text="VM can read Volume data") + cap_write_data = models.BooleanField(default=True, help_text="VM can write Volume data") + cap_host_data = models.BooleanField(default=True, help_text="VM can host Volume data") + + slice_id = models.ManyToManyField(Slice, through="VolumeSlice") + + def __unicode__(self): return self.name + + + @staticmethod + def select_by_user(user): + """ + Only return Volumes accessible by the user. + Admin users can see everything. + """ + if user.is_admin: + qs = Volume.objects.all() + else: + qs = Volume.objects.filter( Q(owner_id=user) | Q(private=False) ) + + return qs + + +class VolumeAccessRight(PlCoreBase): + class Meta: + app_label = "syndicate_storage" + + owner_id = models.ForeignKey(User, verbose_name='user') + + volume = models.ForeignKey(Volume) + + cap_read_data = models.BooleanField(default=True, help_text="VM can read Volume data") + cap_write_data = models.BooleanField(default=True, help_text="VM can write Volume data") + cap_host_data = models.BooleanField(default=True, help_text="VM can host Volume data") + + + def __unicode__(self): return "%s-%s" % (self.owner_id.email, self.volume.name) + + +class ObserverSecretValue( models.TextField ): + class Meta: + app_label = "syndicate_storage" + + __metaclass__ = models.SubfieldBase + + MAGIC_PREFIX = "$SECRET$:" + + @classmethod + def is_encrypted( cls, secret_str ): + # all encrypted secrets start with MAGIC_PREFIX, which is NOT base64-encoded + return secret_str.startswith( cls.MAGIC_PREFIX ) + + @classmethod + def unserialize( cls, serialized_ciphertext ): + # strip prefix and return ciphertext + return serialized_ciphertext[len(cls.MAGIC_PREFIX):] + + @classmethod + def serialize( cls, ciphertext ): + # prepend a magic prefix so we know it's encrypted + return cls.MAGIC_PREFIX + ciphertext + + def to_python( self, secret_str ): + """ + Decrypt the value with the Observer key + """ + + # is this in the clear? + if not ObserverSecretValue.is_encrypted( secret_str ): + # nothing to do + return secret_str + + # otherwise, decrypt it + from syndicate_observer import syndicatelib + + # get observer private key + config = syndicatelib.get_config() + + try: + observer_pkey_path = config.SYNDICATE_PRIVATE_KEY + observer_pkey_pem = syndicatelib.get_private_key_pem( observer_pkey_path ) + except: + raise syndicatelib.SyndicateObserverError( "Internal Syndicate Observer error: failed to load Observer private key" ) + + # deserialize + secret_str = ObserverSecretValue.unserialize( secret_str ) + + # decrypt + if secret_str is not None and len(secret_str) > 0: + + slice_secret = syndicatelib.decrypt_slice_secret( observer_pkey_pem, secret_str ) + + if slice_secret is not None: + return slice_secret + + else: + raise syndicatelib.SyndicateObserverError( "Internal Syndicate Observer error: failed to decrypt slice secret value" ) + else: + return None + + + def pre_save( self, model_inst, add ): + """ + Encrypt the value with the Observer key + """ + + from syndicate_observer import syndicatelib + + # get observer private key + config = syndicatelib.get_config() + + try: + observer_pkey_path = config.SYNDICATE_PRIVATE_KEY + observer_pkey_pem = syndicatelib.get_private_key_pem( observer_pkey_path ) + except: + raise syndicatelib.SyndicateObserverError( "Internal Syndicate Observer error: failed to load Observer private key" ) + + slice_secret = getattr(model_inst, self.attname ) + + if slice_secret is not None: + + # encrypt it + sealed_slice_secret = syndicatelib.encrypt_slice_secret( observer_pkey_pem, slice_secret ) + + return ObserverSecretValue.serialize( sealed_slice_secret ) + + else: + raise syndicatelib.SyndicateObserverError( "Internal Syndicate Observer error: No slice secret generated" ) + + +class SliceSecret(models.Model): # NOTE: not a PlCoreBase + class Meta: + app_label = "syndicate_storage" + + slice_id = models.ForeignKey(Slice) + secret = ObserverSecretValue(blank=True, help_text="Shared secret between OpenCloud and this slice's Syndicate daemons.") + + def __unicode__(self): return self.slice_id.name + + @staticmethod + def select_by_user(user): + """ + Only return slice secrets for slices where this user has 'admin' role. + Admin users can see everything. + """ + if user.is_admin: + qs = SliceSecret.objects.all() + else: + visible_slice_ids = [sp.slice.id for sp in SlicePrivilege.objects.filter(user=user,role__role='admin')] + qs = SliceSecret.objects.filter(slice_id__id__in=visible_slice_ids) + + return qs + + +class VolumeSlice(PlCoreBase): + class Meta: + app_label = "syndicate_storage" + + volume_id = models.ForeignKey(Volume, verbose_name="Volume") + slice_id = models.ForeignKey(Slice, verbose_name="Slice") + + cap_read_data = models.BooleanField(default=True, help_text="VM can read Volume data") + cap_write_data = models.BooleanField(default=True, help_text="VM can write Volume data") + cap_host_data = models.BooleanField(default=True, help_text="VM can host Volume data") + + UG_portnum = models.PositiveIntegerField(help_text="User Gateway port. Any port above 1024 will work, but it must be available slice-wide.", verbose_name="UG port") + RG_portnum = models.PositiveIntegerField(help_text="Replica Gateway port. Any port above 1024 will work, but it must be available slice-wide.", verbose_name="RG port") + + credentials_blob = models.TextField(null=True, blank=True, help_text="Encrypted slice credentials, sealed with the slice secret.") + + def __unicode__(self): return "%s-%s" % (self.volume_id.name, self.slice_id.name) + + def clean(self): + """ + Verify that our fields are in order: + * UG_portnum and RG_portnum have to be valid port numbers between 1025 and 65534 + * UG_portnum and RG_portnum cannot be changed once set. + * UG_portnum and RG_portnum are unique + """ + + if self.UG_portnum == self.RG_portnum: + raise ValidationError( "UG and RG ports must be unique" ) + + if self.UG_portnum < 1025 or self.UG_portnum > 65534: + raise ValidationError( "UG port number must be between 1025 and 65534" ) + + if self.RG_portnum < 1025 or self.RG_portnum > 65534: + raise ValidationError( "RG port number must be between 1025 and 65534" ) + + + def save(self, *args, **kw): + """ + Make sure a SliceSecret exists for this slice + """ + + from syndicate_observer import syndicatelib + + # get observer private key + config = syndicatelib.get_config() + + try: + observer_pkey_path = config.SYNDICATE_PRIVATE_KEY + observer_pkey_pem = syndicatelib.get_private_key_pem( observer_pkey_path ) + except: + raise syndicatelib.SyndicateObserverError( "Internal Syndicate Observer error: failed to load Observer private key" ) + + # get or create the slice secret + slice_secret = syndicatelib.get_or_create_slice_secret( observer_pkey_pem, None, slice_fk=self.slice_id ) + + if slice_secret is None: + raise SyndicateObserverError( "Failed to get or create slice secret for %s" % self.slice_id.name ) + + super(VolumeSlice, self).save(*args, **kw) + + +