From c99f1bd98ff355bd3f63b4929739ea5bd92cd6a3 Mon Sep 17 00:00:00 2001 From: javier Date: Mon, 16 Jun 2014 10:03:06 +0200 Subject: [PATCH] SLA and Service Directory code added --- .gitignore | 1 + manifold/__init__.py | 0 manifold/core/__init__.py | 0 manifold/core/filter.py | 407 ++++ manifold/core/query.py | 585 +++++ manifold/core/result_value.py | 254 +++ manifold/manifoldapi.py | 162 ++ manifold/manifoldproxy.py | 95 + manifold/manifoldresult.py | 61 + manifold/metadata.py | 62 + manifold/static/css/manifold.css | 4 + manifold/static/js/buffer.js | 40 + manifold/static/js/class.js | 64 + manifold/static/js/manifold-query.js | 271 +++ manifold/static/js/manifold.js | 1044 +++++++++ manifold/static/js/metadata.js | 53 + manifold/static/js/plugin.js | 315 +++ manifold/static/js/record_generator.js | 71 + manifold/util/__init__.py | 0 manifold/util/autolog.py | 422 ++++ manifold/util/callback.py | 49 + manifold/util/clause.py | 103 + manifold/util/colors.py | 38 + manifold/util/daemon.py | 343 +++ manifold/util/dfs.py | 95 + manifold/util/enum.py | 7 + manifold/util/frozendict.py | 47 + manifold/util/functional.py | 53 + manifold/util/ipaddr.py | 1897 +++++++++++++++++ manifold/util/log.py | 288 +++ manifold/util/misc.py | 66 + manifold/util/options.py | 95 + manifold/util/plugin_factory.py | 24 + manifold/util/predicate.py | 281 +++ manifold/util/reactor_thread.py | 103 + manifold/util/reactor_wrapper.py | 48 + manifold/util/singleton.py | 19 + manifold/util/storage.py | 29 + manifold/util/type.py | 144 ++ manifold/util/xmldict.py | 77 + myslice/settings.py | 8 + myslice/urls.py | 3 + plugins/queryupdater/__init__.py | 7 +- .../queryupdater/static/js/queryupdater.js | 45 +- plugins/sladialog/__init__.py | 31 + plugins/sladialog/static/css/sladialog.css | 0 plugins/sladialog/static/js/sladialog.js | 208 ++ plugins/sladialog/templates/sladialog.html | 20 + portal/servicedirectory.py | 91 + portal/sliceresourceview.py | 18 + portal/static/css/fed4fire.css | 18 +- portal/templates/_widget-slice-sections.html | 4 +- .../fed4fire_widget-slice-sections.html | 4 +- .../fed4fire/fed4fire_widget-topmenu.html | 1 + portal/templates/servicedirectory.html | 254 +++ portal/templates/slice-resource-view.html | 3 + portal/templates/slice-view.html | 1 + portal/urls.py | 5 +- sample/dashboardview.py | 2 +- sample/pluginview.py | 2 +- sample/scrollview.py | 2 +- sample/tabview.py | 2 +- sla/README | 91 + sla/__init__.py | 0 sla/requirements.txt | 6 + sla/samples/TemplateIMindsService.xml | 55 + sla/samples/providerCreate.bat | 1 + sla/samples/providerIMinds.xml | 5 + sla/samples/simpleAgreementCreation.bat | 1 + .../simpleAgreementCreationParameters.json | 1 + sla/samples/templateCreate.bat | 1 + sla/sla_utils/bin/load-samples.sh | 47 + sla/sla_utils/bin/startagreement.sh | 9 + sla/sla_utils/bin/stopagreement.sh | 9 + sla/sla_utils/samples/agreement03.xml | 55 + sla/sla_utils/samples/agreement04.xml | 55 + sla/sla_utils/samples/enforcement03.xml | 4 + sla/sla_utils/samples/enforcement04.xml | 4 + sla/sla_utils/samples/old/agreement01.xml | 44 + sla/sla_utils/samples/old/agreement02.xml | 77 + sla/sla_utils/samples/old/agreement03_old.xml | 55 + sla/sla_utils/samples/old/agreement04_old.xml | 64 + sla/sla_utils/samples/old/agreement05.xml | 78 + sla/sla_utils/samples/old/agreement05_old.xml | 78 + sla/sla_utils/samples/old/enforcement.xml | 4 + sla/sla_utils/samples/old/enforcement02.xml | 4 + sla/sla_utils/samples/old/enforcement05.xml | 4 + sla/sla_utils/samples/old/template01.xml | 78 + .../samples/provider-virtualwall.xml | 5 + sla/sla_utils/samples/provider-wilab2.xml | 5 + sla/sla_utils/samples/template.xml | 69 + sla/slaclient/__init__.py | 1 + sla/slaclient/restclient.py | 458 ++++ sla/slaclient/restclient_nosecurity.py | 318 +++ sla/slaclient/restclient_old.py | 318 +++ sla/slaclient/service/__init__.py | 0 sla/slaclient/service/fed4fire/__init__.py | 0 .../service/fed4fire/fed4fireservice.py | 138 ++ sla/slaclient/service/fed4fire/jsonparser.py | 120 ++ .../service/fed4fire/tests/__init__.py | 0 .../service/fed4fire/tests/testparsejson.py | 126 ++ .../service/fed4fire/tests/testservice.py | 110 + sla/slaclient/templates/__init__.py | 1 + sla/slaclient/templates/fed4fire/__init__.py | 4 + .../templates/fed4fire/django/__init__.py | 0 .../templates/fed4fire/django/agreement.xml | 47 + .../templates/fed4fire/django/factory.py | 70 + .../templates/fed4fire/django/template.xml | 28 + sla/slaclient/templates/fed4fire/fed4fire.py | 293 +++ .../templates/fed4fire/tests/__init__.py | 0 .../templates/fed4fire/tests/testtemplates.py | 146 ++ sla/slaclient/templates/templates.py | 31 + sla/slaclient/tests/__init__.py | 1 + sla/slaclient/tests/agreement.xml | 81 + sla/slaclient/tests/testconverters.py | 81 + sla/slaclient/wsag_model.py | 233 ++ sla/slaclient/xmlconverter.py | 364 ++++ sla/slicetabsla.py | 356 ++++ sla/static/css/sla.css | 4 + sla/static/js/sla.js | 167 ++ sla/templates/agreement_detail.html | 77 + sla/templates/consumer_agreements.html | 26 + sla/templates/slice-tab-sla.html | 140 ++ sla/templates/slice-tab-sla_alternative.html | 167 ++ sla/templates/violations.html | 23 + sla/templates/violations_template.html | 83 + .../violations_template.sublime-workspace | 191 ++ sla/urls.py | 12 + sla/wsag_helper.py | 116 + 129 files changed, 13676 insertions(+), 10 deletions(-) create mode 100644 manifold/__init__.py create mode 100644 manifold/core/__init__.py create mode 100644 manifold/core/filter.py create mode 100644 manifold/core/query.py create mode 100644 manifold/core/result_value.py create mode 100644 manifold/manifoldapi.py create mode 100644 manifold/manifoldproxy.py create mode 100644 manifold/manifoldresult.py create mode 100644 manifold/metadata.py create mode 100644 manifold/static/css/manifold.css create mode 100644 manifold/static/js/buffer.js create mode 100644 manifold/static/js/class.js create mode 100644 manifold/static/js/manifold-query.js create mode 100644 manifold/static/js/manifold.js create mode 100644 manifold/static/js/metadata.js create mode 100644 manifold/static/js/plugin.js create mode 100644 manifold/static/js/record_generator.js create mode 100644 manifold/util/__init__.py create mode 100644 manifold/util/autolog.py create mode 100644 manifold/util/callback.py create mode 100644 manifold/util/clause.py create mode 100644 manifold/util/colors.py create mode 100644 manifold/util/daemon.py create mode 100644 manifold/util/dfs.py create mode 100644 manifold/util/enum.py create mode 100644 manifold/util/frozendict.py create mode 100644 manifold/util/functional.py create mode 100644 manifold/util/ipaddr.py create mode 100644 manifold/util/log.py create mode 100644 manifold/util/misc.py create mode 100644 manifold/util/options.py create mode 100644 manifold/util/plugin_factory.py create mode 100644 manifold/util/predicate.py create mode 100644 manifold/util/reactor_thread.py create mode 100644 manifold/util/reactor_wrapper.py create mode 100644 manifold/util/singleton.py create mode 100644 manifold/util/storage.py create mode 100644 manifold/util/type.py create mode 100644 manifold/util/xmldict.py create mode 100644 plugins/sladialog/__init__.py create mode 100644 plugins/sladialog/static/css/sladialog.css create mode 100644 plugins/sladialog/static/js/sladialog.js create mode 100644 plugins/sladialog/templates/sladialog.html create mode 100644 portal/servicedirectory.py create mode 100644 portal/templates/servicedirectory.html create mode 100755 sla/README create mode 100755 sla/__init__.py create mode 100755 sla/requirements.txt create mode 100755 sla/samples/TemplateIMindsService.xml create mode 100755 sla/samples/providerCreate.bat create mode 100755 sla/samples/providerIMinds.xml create mode 100755 sla/samples/simpleAgreementCreation.bat create mode 100755 sla/samples/simpleAgreementCreationParameters.json create mode 100755 sla/samples/templateCreate.bat create mode 100755 sla/sla_utils/bin/load-samples.sh create mode 100755 sla/sla_utils/bin/startagreement.sh create mode 100755 sla/sla_utils/bin/stopagreement.sh create mode 100755 sla/sla_utils/samples/agreement03.xml create mode 100755 sla/sla_utils/samples/agreement04.xml create mode 100755 sla/sla_utils/samples/enforcement03.xml create mode 100755 sla/sla_utils/samples/enforcement04.xml create mode 100755 sla/sla_utils/samples/old/agreement01.xml create mode 100755 sla/sla_utils/samples/old/agreement02.xml create mode 100755 sla/sla_utils/samples/old/agreement03_old.xml create mode 100755 sla/sla_utils/samples/old/agreement04_old.xml create mode 100755 sla/sla_utils/samples/old/agreement05.xml create mode 100755 sla/sla_utils/samples/old/agreement05_old.xml create mode 100755 sla/sla_utils/samples/old/enforcement.xml create mode 100755 sla/sla_utils/samples/old/enforcement02.xml create mode 100755 sla/sla_utils/samples/old/enforcement05.xml create mode 100755 sla/sla_utils/samples/old/template01.xml create mode 100755 sla/sla_utils/samples/provider-virtualwall.xml create mode 100755 sla/sla_utils/samples/provider-wilab2.xml create mode 100755 sla/sla_utils/samples/template.xml create mode 100755 sla/slaclient/__init__.py create mode 100755 sla/slaclient/restclient.py create mode 100755 sla/slaclient/restclient_nosecurity.py create mode 100755 sla/slaclient/restclient_old.py create mode 100755 sla/slaclient/service/__init__.py create mode 100755 sla/slaclient/service/fed4fire/__init__.py create mode 100755 sla/slaclient/service/fed4fire/fed4fireservice.py create mode 100755 sla/slaclient/service/fed4fire/jsonparser.py create mode 100755 sla/slaclient/service/fed4fire/tests/__init__.py create mode 100755 sla/slaclient/service/fed4fire/tests/testparsejson.py create mode 100755 sla/slaclient/service/fed4fire/tests/testservice.py create mode 100755 sla/slaclient/templates/__init__.py create mode 100755 sla/slaclient/templates/fed4fire/__init__.py create mode 100755 sla/slaclient/templates/fed4fire/django/__init__.py create mode 100755 sla/slaclient/templates/fed4fire/django/agreement.xml create mode 100755 sla/slaclient/templates/fed4fire/django/factory.py create mode 100755 sla/slaclient/templates/fed4fire/django/template.xml create mode 100755 sla/slaclient/templates/fed4fire/fed4fire.py create mode 100755 sla/slaclient/templates/fed4fire/tests/__init__.py create mode 100755 sla/slaclient/templates/fed4fire/tests/testtemplates.py create mode 100755 sla/slaclient/templates/templates.py create mode 100755 sla/slaclient/tests/__init__.py create mode 100755 sla/slaclient/tests/agreement.xml create mode 100755 sla/slaclient/tests/testconverters.py create mode 100755 sla/slaclient/wsag_model.py create mode 100755 sla/slaclient/xmlconverter.py create mode 100755 sla/slicetabsla.py create mode 100755 sla/static/css/sla.css create mode 100755 sla/static/js/sla.js create mode 100755 sla/templates/agreement_detail.html create mode 100755 sla/templates/consumer_agreements.html create mode 100755 sla/templates/slice-tab-sla.html create mode 100755 sla/templates/slice-tab-sla_alternative.html create mode 100755 sla/templates/violations.html create mode 100755 sla/templates/violations_template.html create mode 100644 sla/templates/violations_template.sublime-workspace create mode 100755 sla/urls.py create mode 100755 sla/wsag_helper.py diff --git a/.gitignore b/.gitignore index bd9a874d..bf4b0b50 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ myslice.ini* foo.* .project .pydevproject +/sla/trashcan \ No newline at end of file diff --git a/manifold/__init__.py b/manifold/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/manifold/core/__init__.py b/manifold/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/manifold/core/filter.py b/manifold/core/filter.py new file mode 100644 index 00000000..3a213483 --- /dev/null +++ b/manifold/core/filter.py @@ -0,0 +1,407 @@ +from types import StringTypes +try: + set +except NameError: + from sets import Set + set = Set + +import time +import datetime # Jordan +#from manifold.util.parameter import Parameter, Mixed, python_type +from manifold.util.predicate import Predicate, eq +from itertools import ifilter + +class Filter(set): + """ + A filter is a set of predicates + """ + + #def __init__(self, s=()): + # super(Filter, self).__init__(s) + + @staticmethod + def from_list(l): + f = Filter() + try: + for element in l: + f.add(Predicate(*element)) + except Exception, e: + print "Error in setting Filter from list", e + return None + return f + + @staticmethod + def from_dict(d): + f = Filter() + for key, value in d.items(): + if key[0] in Predicate.operators.keys(): + f.add(Predicate(key[1:], key[0], value)) + else: + f.add(Predicate(key, '=', value)) + return f + + def to_list(self): + ret = [] + for predicate in self: + ret.append(predicate.to_list()) + return ret + + + @staticmethod + def from_clause(clause): + """ + NOTE: We can only handle simple clauses formed of AND fields. + """ + raise Exception, "Not implemented" + + def filter_by(self, predicate): + self.add(predicate) + return self + + def __str__(self): + return ' AND '.join([str(pred) for pred in self]) + + def __repr__(self): + return '' % ' AND '.join([str(pred) for pred in self]) + + def __key(self): + return tuple([hash(pred) for pred in self]) + + def __hash__(self): + return hash(self.__key()) + + def __additem__(self, value): + if value.__class__ != Predicate: + raise TypeError("Element of class Predicate expected, received %s" % value.__class__.__name__) + set.__additem__(self, value) + + def keys(self): + return set([x.key for x in self]) + + # XXX THESE FUNCTIONS SHOULD ACCEPT MULTIPLE FIELD NAMES + + def has(self, key): + for x in self: + if x.key == key: + return True + return False + + def has_op(self, key, op): + for x in self: + if x.key == key and x.op == op: + return True + return False + + def has_eq(self, key): + return self.has_op(key, eq) + + def get(self, key): + ret = [] + for x in self: + if x.key == key: + ret.append(x) + return ret + + def delete(self, key): + to_del = [] + for x in self: + if x.key == key: + to_del.append(x) + for x in to_del: + self.remove(x) + + #self = filter(lambda x: x.key != key, self) + + def get_op(self, key, op): + if isinstance(op, (list, tuple, set)): + for x in self: + if x.key == key and x.op in op: + return x.value + else: + for x in self: + if x.key == key and x.op == op: + return x.value + return None + + def get_eq(self, key): + return self.get_op(key, eq) + + def set_op(self, key, op, value): + for x in self: + if x.key == key and x.op == op: + x.value = value + return + raise KeyError, key + + def set_eq(self, key, value): + return self.set_op(key, eq, value) + + def get_predicates(self, key): + # XXX Would deserve returning a filter (cf usage in SFA gateway) + ret = [] + for x in self: + if x.key == key: + ret.append(x) + return ret + +# def filter(self, dic): +# # We go through every filter sequentially +# for predicate in self: +# print "predicate", predicate +# dic = predicate.filter(dic) +# return dic + + def match(self, dic, ignore_missing=True): + for predicate in self: + if not predicate.match(dic, ignore_missing): + return False + return True + + def filter(self, l): + output = [] + for x in l: + if self.match(x): + output.append(x) + return output + + def get_field_names(self): + field_names = set() + for predicate in self: + field_names |= predicate.get_field_names() + return field_names + +#class OldFilter(Parameter, dict): +# """ +# A type of parameter that represents a filter on one or more +# columns of a database table. +# Special features provide support for negation, upper and lower bounds, +# as well as sorting and clipping. +# +# +# fields should be a dictionary of field names and types. +# As of PLCAPI-4.3-26, we provide support for filtering on +# sequence types as well, with the special '&' and '|' modifiers. +# example : fields = {'node_id': Parameter(int, "Node identifier"), +# 'hostname': Parameter(int, "Fully qualified hostname", max = 255), +# ...} +# +# +# filter should be a dictionary of field names and values +# representing the criteria for filtering. +# example : filter = { 'hostname' : '*.edu' , site_id : [34,54] } +# Whether the filter represents an intersection (AND) or a union (OR) +# of these criteria is determined by the join_with argument +# provided to the sql method below +# +# Special features: +# +# * a field starting with '&' or '|' should refer to a sequence type +# the semantic is then that the object value (expected to be a list) +# should contain all (&) or any (|) value specified in the corresponding +# filter value. See other examples below. +# example : filter = { '|role_ids' : [ 20, 40 ] } +# example : filter = { '|roles' : ['tech', 'pi'] } +# example : filter = { '&roles' : ['admin', 'tech'] } +# example : filter = { '&roles' : 'tech' } +# +# * a field starting with the ~ character means negation. +# example : filter = { '~peer_id' : None } +# +# * a field starting with < [ ] or > means lower than or greater than +# < > uses strict comparison +# [ ] is for using <= or >= instead +# example : filter = { ']event_id' : 2305 } +# example : filter = { '>time' : 1178531418 } +# in this example the integer value denotes a unix timestamp +# +# * if a value is a sequence type, then it should represent +# a list of possible values for that field +# example : filter = { 'node_id' : [12,34,56] } +# +# * a (string) value containing either a * or a % character is +# treated as a (sql) pattern; * are replaced with % that is the +# SQL wildcard character. +# example : filter = { 'hostname' : '*.jp' } +# +# * the filter's keys starting with '-' are special and relate to sorting and clipping +# * '-SORT' : a field name, or an ordered list of field names that are used for sorting +# these fields may start with + (default) or - for denoting increasing or decreasing order +# example : filter = { '-SORT' : [ '+node_id', '-hostname' ] } +# * '-OFFSET' : the number of first rows to be ommitted +# * '-LIMIT' : the amount of rows to be returned +# example : filter = { '-OFFSET' : 100, '-LIMIT':25} +# +# Here are a few realistic examples +# +# GetNodes ( { 'node_type' : 'regular' , 'hostname' : '*.edu' , '-SORT' : 'hostname' , '-OFFSET' : 30 , '-LIMIT' : 25 } ) +# would return regular (usual) nodes matching '*.edu' in alphabetical order from 31th to 55th +# +# GetPersons ( { '|role_ids' : [ 20 , 40] } ) +# would return all persons that have either pi (20) or tech (40) roles +# +# GetPersons ( { '&role_ids' : 10 } ) +# GetPersons ( { '&role_ids' : 10 } ) +# GetPersons ( { '|role_ids' : [ 10 ] } ) +# GetPersons ( { '|role_ids' : [ 10 ] } ) +# all 4 forms are equivalent and would return all admin users in the system +# """ +# +# def __init__(self, fields = {}, filter = {}, doc = "Attribute filter"): +# # Store the filter in our dict instance +# dict.__init__(self, filter) +# +# # Declare ourselves as a type of parameter that can take +# # either a value or a list of values for each of the specified +# # fields. +# self.fields = dict ( [ ( field, Mixed (expected, [expected])) +# for (field,expected) in fields.iteritems() ] ) +# +# # Null filter means no filter +# Parameter.__init__(self, self.fields, doc = doc, nullok = True) +# +# def sql(self, api, join_with = "AND"): +# """ +# Returns a SQL conditional that represents this filter. +# """ +# +# # So that we always return something +# if join_with == "AND": +# conditionals = ["True"] +# elif join_with == "OR": +# conditionals = ["False"] +# else: +# assert join_with in ("AND", "OR") +# +# # init +# sorts = [] +# clips = [] +# +# for field, value in self.iteritems(): +# # handle negation, numeric comparisons +# # simple, 1-depth only mechanism +# +# modifiers={'~' : False, +# '<' : False, '>' : False, +# '[' : False, ']' : False, +# '-' : False, +# '&' : False, '|' : False, +# '{': False , +# } +# def check_modifiers(field): +# if field[0] in modifiers.keys(): +# modifiers[field[0]] = True +# field = field[1:] +# return check_modifiers(field) +# return field +# field = check_modifiers(field) +# +# # filter on fields +# if not modifiers['-']: +# if field not in self.fields: +# raise PLCInvalidArgument, "Invalid filter field '%s'" % field +# +# # handling array fileds always as compound values +# if modifiers['&'] or modifiers['|']: +# if not isinstance(value, (list, tuple, set)): +# value = [value,] +# +# if isinstance(value, (list, tuple, set)): +# # handling filters like '~slice_id':[] +# # this should return true, as it's the opposite of 'slice_id':[] which is false +# # prior to this fix, 'slice_id':[] would have returned ``slice_id IN (NULL) '' which is unknown +# # so it worked by coincidence, but the negation '~slice_ids':[] would return false too +# if not value: +# if modifiers['&'] or modifiers['|']: +# operator = "=" +# value = "'{}'" +# else: +# field="" +# operator="" +# value = "FALSE" +# else: +# value = map(str, map(api.db.quote, value)) +# if modifiers['&']: +# operator = "@>" +# value = "ARRAY[%s]" % ", ".join(value) +# elif modifiers['|']: +# operator = "&&" +# value = "ARRAY[%s]" % ", ".join(value) +# else: +# operator = "IN" +# value = "(%s)" % ", ".join(value) +# else: +# if value is None: +# operator = "IS" +# value = "NULL" +# elif isinstance(value, StringTypes) and \ +# (value.find("*") > -1 or value.find("%") > -1): +# operator = "LIKE" +# # insert *** in pattern instead of either * or % +# # we dont use % as requests are likely to %-expansion later on +# # actual replacement to % done in PostgreSQL.py +# value = value.replace ('*','***') +# value = value.replace ('%','***') +# value = str(api.db.quote(value)) +# else: +# operator = "=" +# if modifiers['<']: +# operator='<' +# if modifiers['>']: +# operator='>' +# if modifiers['[']: +# operator='<=' +# if modifiers[']']: +# operator='>=' +# #else: +# # value = str(api.db.quote(value)) +# # jordan +# if isinstance(value, StringTypes) and value[-2:] != "()": # XXX +# value = str(api.db.quote(value)) +# if isinstance(value, datetime.datetime): +# value = str(api.db.quote(str(value))) +# +# #if prefix: +# # field = "%s.%s" % (prefix,field) +# if field: +# clause = "\"%s\" %s %s" % (field, operator, value) +# else: +# clause = "%s %s %s" % (field, operator, value) +# +# if modifiers['~']: +# clause = " ( NOT %s ) " % (clause) +# +# conditionals.append(clause) +# # sorting and clipping +# else: +# if field not in ('SORT','OFFSET','LIMIT'): +# raise PLCInvalidArgument, "Invalid filter, unknown sort and clip field %r"%field +# # sorting +# if field == 'SORT': +# if not isinstance(value,(list,tuple,set)): +# value=[value] +# for field in value: +# order = 'ASC' +# if field[0] == '+': +# field = field[1:] +# elif field[0] == '-': +# field = field[1:] +# order = 'DESC' +# if field not in self.fields: +# raise PLCInvalidArgument, "Invalid field %r in SORT filter"%field +# sorts.append("%s %s"%(field,order)) +# # clipping +# elif field == 'OFFSET': +# clips.append("OFFSET %d"%value) +# # clipping continued +# elif field == 'LIMIT' : +# clips.append("LIMIT %d"%value) +# +# where_part = (" %s " % join_with).join(conditionals) +# clip_part = "" +# if sorts: +# clip_part += " ORDER BY " + ",".join(sorts) +# if clips: +# clip_part += " " + " ".join(clips) +## print 'where_part=',where_part,'clip_part',clip_part +# return (where_part,clip_part) +# diff --git a/manifold/core/query.py b/manifold/core/query.py new file mode 100644 index 00000000..976f4978 --- /dev/null +++ b/manifold/core/query.py @@ -0,0 +1,585 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Query representation +# +# Copyright (C) UPMC Paris Universitas +# Authors: +# Jordan Augé +# Marc-Olivier Buob +# Thierry Parmentelat + +from types import StringTypes +from manifold.core.filter import Filter, Predicate +from manifold.util.frozendict import frozendict +from manifold.util.type import returns, accepts +from manifold.util.clause import Clause +import copy + +import json +import uuid + +def uniqid (): + return uuid.uuid4().hex + +debug=False +#debug=True + +class ParameterError(StandardError): pass + +class Query(object): + """ + Implements a TopHat query. + + We assume this is a correct DAG specification. + + 1/ A field designates several tables = OR specification. + 2/ The set of fields specifies a AND between OR clauses. + """ + + #--------------------------------------------------------------------------- + # Constructor + #--------------------------------------------------------------------------- + + def __init__(self, *args, **kwargs): + + self.query_uuid = uniqid() + + # Initialize optional parameters + self.clear() + + #l = len(kwargs.keys()) + len_args = len(args) + + if len(args) == 1: + if isinstance(args[0], dict): + kwargs = args[0] + args = [] + + # Initialization from a tuple + + if len_args in range(2, 7) and type(args) == tuple: + # Note: range(x,y) <=> [x, y[ + + # XXX UGLY + if len_args == 3: + self.action = 'get' + self.params = {} + self.timestamp = 'now' + self.object, self.filters, self.fields = args + elif len_args == 4: + self.object, self.filters, self.params, self.fields = args + self.action = 'get' + self.timestamp = 'now' + else: + self.action, self.object, self.filters, self.params, self.fields, self.timestamp = args + + # Initialization from a dict + elif "object" in kwargs: + if "action" in kwargs: + self.action = kwargs["action"] + del kwargs["action"] + else: + print "W: defaulting to get action" + self.action = "get" + + + self.object = kwargs["object"] + del kwargs["object"] + + if "filters" in kwargs: + self.filters = kwargs["filters"] + del kwargs["filters"] + else: + self.filters = Filter() + + if "fields" in kwargs: + self.fields = set(kwargs["fields"]) + del kwargs["fields"] + else: + self.fields = set() + + # "update table set x = 3" => params == set + if "params" in kwargs: + self.params = kwargs["params"] + del kwargs["params"] + else: + self.params = {} + + if "timestamp" in kwargs: + self.timestamp = kwargs["timestamp"] + del kwargs["timestamp"] + else: + self.timestamp = "now" + + if kwargs: + raise ParameterError, "Invalid parameter(s) : %r" % kwargs.keys() + #else: + # raise ParameterError, "No valid constructor found for %s : args = %r" % (self.__class__.__name__, args) + + self.sanitize() + + def sanitize(self): + if not self.filters: self.filters = Filter() + if not self.params: self.params = {} + if not self.fields: self.fields = set() + if not self.timestamp: self.timestamp = "now" + + if isinstance(self.filters, list): + f = self.filters + self.filters = Filter() + for x in f: + pred = Predicate(x) + self.filters.add(pred) + elif isinstance(self.filters, Clause): + self.filters = Filter.from_clause(self.filters) + + if isinstance(self.fields, list): + self.fields = set(self.fields) + + for field in self.fields: + if not isinstance(field, StringTypes): + raise TypeError("Invalid field name %s (string expected, got %s)" % (field, type(field))) + + #--------------------------------------------------------------------------- + # Helpers + #--------------------------------------------------------------------------- + + def copy(self): + return copy.deepcopy(self) + + def clear(self): + self.action = 'get' + self.object = None + self.filters = Filter() + self.params = {} + self.fields = set() + self.timestamp = 'now' # ignored for now + + def to_sql(self, platform='', multiline=False): + get_params_str = lambda : ', '.join(['%s = %r' % (k, v) for k, v in self.get_params().items()]) + get_select_str = lambda : ', '.join(self.get_select()) + + table = self.get_from() + select = 'SELECT %s' % (get_select_str() if self.get_select() else '*') + where = 'WHERE %s' % self.get_where() if self.get_where() else '' + at = 'AT %s' % self.get_timestamp() if self.get_timestamp() else '' + params = 'SET %s' % get_params_str() if self.get_params() else '' + + sep = ' ' if not multiline else '\n ' + if platform: platform = "%s:" % platform + strmap = { + 'get' : '%(select)s%(sep)s%(at)s%(sep)sFROM %(platform)s%(table)s%(sep)s%(where)s%(sep)s', + 'update': 'UPDATE %(platform)s%(table)s%(sep)s%(params)s%(sep)s%(where)s%(sep)s%(select)s', + 'create': 'INSERT INTO %(platform)s%(table)s%(sep)s%(params)s%(sep)s%(select)s', + 'delete': 'DELETE FROM %(platform)s%(table)s%(sep)s%(where)s' + } + + return strmap[self.action] % locals() + + @returns(StringTypes) + def __str__(self): + return self.to_sql(multiline=True) + + @returns(StringTypes) + def __repr__(self): + return self.to_sql() + + def __key(self): + return (self.action, self.object, self.filters, frozendict(self.params), frozenset(self.fields)) + + def __hash__(self): + return hash(self.__key()) + + #--------------------------------------------------------------------------- + # Conversion + #--------------------------------------------------------------------------- + + def to_dict(self): + return { + 'action': self.action, + 'object': self.object, + 'timestamp': self.timestamp, + 'filters': self.filters.to_list(), + 'params': self.params, + 'fields': list(self.fields) + } + + def to_json (self, analyzed_query=None): + query_uuid=self.query_uuid + a=self.action + o=self.object + t=self.timestamp + f=json.dumps (self.filters.to_list()) + p=json.dumps (self.params) + c=json.dumps (list(self.fields)) + # xxx unique can be removed, but for now we pad the js structure + unique=0 + + if not analyzed_query: + aq = 'null' + else: + aq = analyzed_query.to_json() + sq="{}" + + result= """ new ManifoldQuery('%(a)s', '%(o)s', '%(t)s', %(f)s, %(p)s, %(c)s, %(unique)s, '%(query_uuid)s', %(aq)s, %(sq)s)"""%locals() + if debug: print 'ManifoldQuery.to_json:',result + return result + + # this builds a ManifoldQuery object from a dict as received from javascript through its ajax request + # we use a json-encoded string - see manifold.js for the sender part + # e.g. here's what I captured from the server's output + # manifoldproxy.proxy: request.POST + def fill_from_POST (self, POST_dict): + try: + json_string=POST_dict['json'] + dict=json.loads(json_string) + for (k,v) in dict.iteritems(): + setattr(self,k,v) + except: + print "Could not decode incoming ajax request as a Query, POST=",POST_dict + if (debug): + import traceback + traceback.print_exc() + self.sanitize() + + #--------------------------------------------------------------------------- + # Accessors + #--------------------------------------------------------------------------- + + @returns(StringTypes) + def get_action(self): + return self.action + + @returns(frozenset) + def get_select(self): + return frozenset(self.fields) + + @returns(StringTypes) + def get_from(self): + return self.object + + @returns(Filter) + def get_where(self): + return self.filters + + @returns(dict) + def get_params(self): + return self.params + + @returns(StringTypes) + def get_timestamp(self): + return self.timestamp + +#DEPRECATED# +#DEPRECATED# def make_filters(self, filters): +#DEPRECATED# return Filter(filters) +#DEPRECATED# +#DEPRECATED# def make_fields(self, fields): +#DEPRECATED# if isinstance(fields, (list, tuple)): +#DEPRECATED# return set(fields) +#DEPRECATED# else: +#DEPRECATED# raise Exception, "Invalid field specification" + + #--------------------------------------------------------------------------- + # LINQ-like syntax + #--------------------------------------------------------------------------- + + @classmethod + #@returns(Query) + def action(self, action, object): + """ + (Internal usage). Craft a Query according to an action name + See methods: get, update, delete, execute. + Args: + action: A String among {"get", "update", "delete", "execute"} + object: The name of the queried object (String) + Returns: + The corresponding Query instance + """ + query = Query() + query.action = action + query.object = object + return query + + @classmethod + #@returns(Query) + def get(self, object): + """ + Craft the Query which fetches the records related to a given object + Args: + object: The name of the queried object (String) + Returns: + The corresponding Query instance + """ + return self.action("get", object) + + @classmethod + #@returns(Query) + def update(self, object): + """ + Craft the Query which updates the records related to a given object + Args: + object: The name of the queried object (String) + Returns: + The corresponding Query instance + """ + return self.action("update", object) + + @classmethod + #@returns(Query) + def create(self, object): + """ + Craft the Query which create the records related to a given object + Args: + object: The name of the queried object (String) + Returns: + The corresponding Query instance + """ + return self.action("create", object) + + @classmethod + #@returns(Query) + def delete(self, object): + """ + Craft the Query which delete the records related to a given object + Args: + object: The name of the queried object (String) + Returns: + The corresponding Query instance + """ + return self.action("delete", object) + + @classmethod + #@returns(Query) + def execute(self, object): + """ + Craft the Query which execute a processing related to a given object + Args: + object: The name of the queried object (String) + Returns: + The corresponding Query instance + """ + return self.action("execute", object) + + #@returns(Query) + def at(self, timestamp): + """ + Set the timestamp carried by the query + Args: + timestamp: The timestamp (it may be a python timestamp, a string + respecting the "%Y-%m-%d %H:%M:%S" python format, or "now") + Returns: + The self Query instance + """ + self.timestamp = timestamp + return self + + def filter_by(self, *args): + """ + Args: + args: It may be: + - the parts of a Predicate (key, op, value) + - None + - a Filter instance + - a set/list/tuple of Predicate instances + """ + if len(args) == 1: + filters = args[0] + if filters == None: + self.filters = Filter() + return self + if not isinstance(filters, (set, list, tuple, Filter)): + filters = [filters] + for predicate in filters: + self.filters.add(predicate) + elif len(args) == 3: + predicate = Predicate(*args) + self.filters.add(predicate) + else: + raise Exception, 'Invalid expression for filter' + return self + + def select(self, *fields): + + # Accept passing iterables + if len(fields) == 1: + tmp, = fields + if not tmp: + fields = None + elif isinstance(tmp, (list, tuple, set, frozenset)): + fields = tuple(tmp) + + if not fields: + # Delete all fields + self.fields = set() + return self + + for field in fields: + self.fields.add(field) + return self + + def set(self, params): + self.params.update(params) + return self + + def __or__(self, query): + assert self.action == query.action + assert self.object == query.object + assert self.timestamp == query.timestamp # XXX + filter = self.filters | query.filters + # fast dict union + # http://my.safaribooksonline.com/book/programming/python/0596007973/python-shortcuts/pythoncook2-chp-4-sect-17 + params = dict(self.params, **query.params) + fields = self.fields | query.fields + return Query.action(self.action, self.object).filter_by(filter).select(fields) + + def __and__(self, query): + assert self.action == query.action + assert self.object == query.object + assert self.timestamp == query.timestamp # XXX + filter = self.filters & query.filters + # fast dict intersection + # http://my.safaribooksonline.com/book/programming/python/0596007973/python-shortcuts/pythoncook2-chp-4-sect-17 + params = dict.fromkeys([x for x in self.params if x in query.params]) + fields = self.fields & query.fields + return Query.action(self.action, self.object).filter_by(filter).select(fields) + + def __le__(self, query): + return ( self == self & query ) or ( query == self | query ) + +class AnalyzedQuery(Query): + + # XXX we might need to propagate special parameters sur as DEBUG, etc. + + def __init__(self, query=None, metadata=None): + self.clear() + self.metadata = metadata + if query: + self.query_uuid = query.query_uuid + self.analyze(query) + else: + self.query_uuid = uniqid() + + @returns(StringTypes) + def __str__(self): + out = [] + fields = self.get_select() + fields = ", ".join(fields) if fields else '*' + out.append("SELECT %s FROM %s WHERE %s" % ( + fields, + self.get_from(), + self.get_where() + )) + cpt = 1 + for method, subquery in self.subqueries(): + out.append(' [SQ #%d : %s] %s' % (cpt, method, str(subquery))) + cpt += 1 + + return "\n".join(out) + + def clear(self): + super(AnalyzedQuery, self).clear() + self._subqueries = {} + + def subquery(self, method): + # Allows for the construction of a subquery + if not method in self._subqueries: + analyzed_query = AnalyzedQuery(metadata=self.metadata) + analyzed_query.action = self.action + try: + type = self.metadata.get_field_type(self.object, method) + except ValueError ,e: # backwards 1..N + type = method + analyzed_query.object = type + self._subqueries[method] = analyzed_query + return self._subqueries[method] + + def get_subquery(self, method): + return self._subqueries.get(method, None) + + def remove_subquery(self, method): + del self._subqueries[method] + + def get_subquery_names(self): + return set(self._subqueries.keys()) + + def get_subqueries(self): + return self._subqueries + + def subqueries(self): + for method, subquery in self._subqueries.iteritems(): + yield (method, subquery) + + def filter_by(self, filters): + if not isinstance(filters, (set, list, tuple, Filter)): + filters = [filters] + for predicate in filters: + if predicate and '.' in predicate.key: + method, subkey = predicate.key.split('.', 1) + # Method contains the name of the subquery, we need the type + # XXX type = self.metadata.get_field_type(self.object, method) + sub_pred = Predicate(subkey, predicate.op, predicate.value) + self.subquery(method).filter_by(sub_pred) + else: + super(AnalyzedQuery, self).filter_by(predicate) + return self + + def select(self, *fields): + + # XXX passing None should reset fields in all subqueries + + # Accept passing iterables + if len(fields) == 1: + tmp, = fields + if isinstance(tmp, (list, tuple, set, frozenset)): + fields = tuple(tmp) + + for field in fields: + if field and '.' in field: + method, subfield = field.split('.', 1) + # Method contains the name of the subquery, we need the type + # XXX type = self.metadata.get_field_type(self.object, method) + self.subquery(method).select(subfield) + else: + super(AnalyzedQuery, self).select(field) + return self + + def set(self, params): + for param, value in self.params.items(): + if '.' in param: + method, subparam = param.split('.', 1) + # Method contains the name of the subquery, we need the type + # XXX type = self.metadata.get_field_type(self.object, method) + self.subquery(method).set({subparam: value}) + else: + super(AnalyzedQuery, self).set({param: value}) + return self + + def analyze(self, query): + self.clear() + self.action = query.action + self.object = query.object + self.filter_by(query.filters) + self.set(query.params) + self.select(query.fields) + + def to_json (self): + query_uuid=self.query_uuid + a=self.action + o=self.object + t=self.timestamp + f=json.dumps (self.filters.to_list()) + p=json.dumps (self.params) + c=json.dumps (list(self.fields)) + # xxx unique can be removed, but for now we pad the js structure + unique=0 + + aq = 'null' + sq=", ".join ( [ "'%s':%s" % (object, subquery.to_json()) + for (object, subquery) in self._subqueries.iteritems()]) + sq="{%s}"%sq + + result= """ new ManifoldQuery('%(a)s', '%(o)s', '%(t)s', %(f)s, %(p)s, %(c)s, %(unique)s, '%(query_uuid)s', %(aq)s, %(sq)s)"""%locals() + if debug: print 'ManifoldQuery.to_json:',result + return result diff --git a/manifold/core/result_value.py b/manifold/core/result_value.py new file mode 100644 index 00000000..4fe505f8 --- /dev/null +++ b/manifold/core/result_value.py @@ -0,0 +1,254 @@ +# Inspired from GENI error codes + +import time +import pprint + +class ResultValue(dict): + + # type + SUCCESS = 0 + WARNING = 1 + ERROR = 2 + + # origin + CORE = 0 + GATEWAY = 1 + + # code + SUCCESS = 0 + SERVERBUSY = 32001 + BADARGS = 1 + ERROR = 2 + FORBIDDEN = 3 + BADVERSION = 4 + SERVERERROR = 5 + TOOBIG = 6 + REFUSED = 7 + TIMEDOUT = 8 + DBERROR = 9 + RPCERROR = 10 + + # description + ERRSTR = { + SUCCESS : 'Success', + SERVERBUSY : 'Server is (temporarily) too busy; try again later', + BADARGS : 'Bad Arguments: malformed', + ERROR : 'Error (other)', + FORBIDDEN : 'Operation Forbidden: eg supplied credentials do not provide sufficient privileges (on the given slice)', + BADVERSION : 'Bad Version (eg of RSpec)', + SERVERERROR : 'Server Error', + TOOBIG : 'Too Big (eg request RSpec)', + REFUSED : 'Operation Refused', + TIMEDOUT : 'Operation Timed Out', + DBERROR : 'Database Error', + RPCERROR : '' + } + + ALLOWED_FIELDS = set(['origin', 'type', 'code', 'value', 'description', 'traceback', 'ts']) + + def __init__(self, **kwargs): + + # Checks + given = set(kwargs.keys()) + cstr_success = set(['code', 'origin', 'value']) <= given + cstr_error = set(['code', 'type', 'origin', 'description']) <= given + assert given <= self.ALLOWED_FIELDS, "Wrong fields in ResultValue constructor: %r" % (given - self.ALLOWED_FIELDS) + assert cstr_success or cstr_error, 'Incomplete set of fields in ResultValue constructor: %r' % given + + dict.__init__(self, **kwargs) + + # Set missing fields to None + for field in self.ALLOWED_FIELDS - given: + self[field] = None + if not 'ts' in self: + self['ts'] = time.time() + + + # Internal MySlice errors : return ERROR + # Internal MySlice warnings : return RESULT WITH WARNINGS + # Debug : add DEBUG INFORMATION + # Gateway errors : return RESULT WITH WARNING + # all Gateways errors : return ERROR + + @classmethod + def get_result_value(self, results, result_value_array): + # let's analyze the results of the query plan + # XXX we should inspect all errors to determine whether to return a + # result or not + if not result_value_array: + # No error + return ResultValue(code=self.SUCCESS, origin=[self.CORE, 0], value=results) + else: + # Handle errors + return ResultValue(code=self.WARNING, origin=[self.CORE, 0], description=result_value_array, value=results) + + @classmethod + def get_error(self, error): + return ResultValue(code=error, origin=[self.CORE, 0], value=self.ERRSTR[error]) + + @classmethod + def get_success(self, result): + return ResultValue(code=self.SUCCESS, origin=[self.CORE, 0], value=result) + + def ok_value(self): + return self['value'] + + def error(self): + err = "%r" % self['description'] + + @staticmethod + def to_html (raw_dict): + return pprint.pformat (raw_dict).replace("\\n","
") + +# 67 +# 68 9 +# 69 +# 70 Database Error +# 71 +# 72 +# 73 10 +# 74 +# 75 RPC Error +# 76 +# 77 +# 78 11 +# 79 +# 80 Unavailable (eg server in lockdown) +# 81 +# 82 +# 83 12 +# 84 +# 85 Search Failed (eg for slice) +# 86 +# 87 +# 88 13 +# 89 +# 90 Operation Unsupported +# 91 +# 92 +# 93 14 +# 94 +# 95 Busy (resource, slice, or server); try again +# later +# 96 +# 97 +# 98 15 +# 99 +# 100 Expired (eg slice) +# 101 +# 102 +# 103 16 +# 104 +# 105 In Progress +# 106 +# 107 +# 108 17 +# 109 +# 110 Already Exists (eg slice) +# 111 +# 112 +# 114 +# 115 18 +# 116 +# 117 Required argument(s) missing +# 118 +# 119 +# 120 19 +# 121 +# 122 Input Argument outside of legal range +# 123 +# 124 +# 125 20 +# 126 +# 127 Not authorized: Supplied credential is +# invalid +# 128 +# 129 +# 130 21 +# 131 +# 132 Not authorized: Supplied credential expired +# 133 +# 134 +# 135 22 +# 136 +# 137 Not authorized: Supplied credential does not match client +# certificate or does not match the given slice URN +# 138 +# 139 +# 140 23 +# 141 +# 142 Not authorized: Supplied credential not signed by a trusted +# authority +# 143 +# 144 +# 145 24 +# 146 +# 147 VLAN tag(s) requested not available (likely stitching +# failure) +# 148 +# 149 +# 150 +# diff --git a/manifold/manifoldapi.py b/manifold/manifoldapi.py new file mode 100644 index 00000000..b1a1a0cc --- /dev/null +++ b/manifold/manifoldapi.py @@ -0,0 +1,162 @@ +# Manifold API Python interface +import copy, xmlrpclib + +from myslice.configengine import ConfigEngine + +from django.contrib import messages +from manifoldresult import ManifoldResult, ManifoldCode, ManifoldException +from manifold.core.result_value import ResultValue + +debug=False +debug=True +debug_deep=False +#debug_deep=True + +########## ugly stuff for hopefully nicer debug messages +def mytruncate (obj, l): + # we will add '..' + l1=l-2 + repr="%s"%obj + return (repr[:l1]+'..') if len(repr)>l1 else repr + +from time import time, gmtime, strftime +from math import trunc +def mytime (start=None): + gm=gmtime() + t=time() + msg=strftime("%H:%M:%S-", gmtime())+"%03d"%((t-trunc(t))*1000) + if start is not None: msg += " (%03fs)"%(t-start) + return t,msg +########## + +class ManifoldAPI: + + def __init__ (self, auth=None, cainfo=None): + + self.auth = auth + self.cainfo = cainfo + self.errors = [] + self.trace = [] + self.calls = {} + self.multicall = False + self.url = ConfigEngine().manifold_url() + self.server = xmlrpclib.Server(self.url, verbose=False, allow_none=True) + + def __repr__ (self): return "ManifoldAPI[%s]"%self.url + + def _print_value (self, value): + print "+++",'value', + if isinstance (value,list): print "[%d]"%len(value), + elif isinstance (value,dict): print "{%d}"%len(value), + print mytruncate (value,80) + + # a one-liner to give a hint of what the return value looks like + def _print_result (self, result): + if not result: print "[no/empty result]" + elif isinstance (result,str): print "result is '%s'"%result + elif isinstance (result,list): print "result is a %d-elts list"%len(result) + elif isinstance (result,dict): + print "result is a dict with %d keys : %s"%(len(result),result.keys()) + for (k,v) in result.iteritems(): + if v is None: continue + if k=='value': self._print_value(v) + else: print '+++',k,':',mytruncate (v,30) + else: print "[dont know how to display result] %s"%result + + # how to display a call + def _repr_query (self,methodName, query): + try: action=query['action'] + except: action="???" + try: subject=query['object'] + except: subject="???" + # most of the time, we run 'forward' + if methodName=='forward': return "forward(%s(%s))"%(action,subject) + else: return "%s(%s)"%(action,subject) + + # xxx temporary code for scaffolding a ManifolResult on top of an API that does not expose error info + # as of march 2013 we work with an API that essentially either returns the value, or raises + # an xmlrpclib.Fault exception with always the same 8002 code + # since most of the time we're getting this kind of issues for expired sessions + # (looks like sessions are rather short-lived), for now the choice is to map these errors on + # a SESSION_EXPIRED code + def __getattr__(self, methodName): + def func(*args, **kwds): + # shorthand + def repr(): return self._repr_query (methodName, args[0]) + try: + if debug: + start,msg = mytime() + print "====>",msg,"ManifoldAPI.%s"%repr(),"url",self.url + # No password in the logs + logAuth = copy.copy(self.auth) + for obfuscate in ['Authring','session']: + if obfuscate in logAuth: logAuth[obfuscate]="XXX" + if debug_deep: print "=> auth",logAuth + if debug_deep: print "=> args",args,"kwds",kwds + annotations = { + 'authentication': self.auth + } + args += (annotations,) + result=getattr(self.server, methodName)(*args, **kwds) + print "%s%r" %(methodName, args) + + if debug: + print '<= result=', + self._print_result(result) + end,msg = mytime(start) + print "<====",msg,"backend call %s returned"%(repr()) + + return ResultValue(**result) + + except Exception,error: + print "** MANIFOLD API ERROR **" + if debug: + print "===== xmlrpc catch-all exception:",error + import traceback + traceback.print_exc(limit=3) + if "Connection refused" in error: + raise ManifoldException ( ManifoldResult (code=ManifoldCode.SERVER_UNREACHABLE, + output="%s answered %s"%(self.url,error))) + # otherwise + print "<==== ERROR On ManifoldAPI.%s"%repr() + raise ManifoldException ( ManifoldResult (code=ManifoldCode.SERVER_UNREACHABLE, output="%s"%error) ) + + return func + +def _execute_query(request, query, manifold_api_session_auth): + manifold_api = ManifoldAPI(auth=manifold_api_session_auth) + print "-"*80 + print query + print query.to_dict() + print "-"*80 + result = manifold_api.forward(query.to_dict()) + if result['code'] == 2: + # this is gross; at the very least we need to logout() + # but most importantly there is a need to refine that test, since + # code==2 does not necessarily mean an expired session + # XXX only if we know it is the issue + del request.session['manifold'] + # Flush django session + request.session.flush() + #raise Exception, 'Error running query: %r' % result + + if result['code'] == 1: + print "WARNING" + print result['description'] + + # XXX Handle errors + #Error running query: {'origin': [0, 'XMLRPCAPI'], 'code': 2, 'description': 'No such session: No row was found for one()', 'traceback': 'Traceback (most recent call last):\n File "/usr/local/lib/python2.7/dist-packages/manifold/core/xmlrpc_api.py", line 68, in xmlrpc_forward\n user = Auth(auth).check()\n File "/usr/local/lib/python2.7/dist-packages/manifold/auth/__init__.py", line 245, in check\n return self.auth_method.check()\n File "/usr/local/lib/python2.7/dist-packages/manifold/auth/__init__.py", line 95, in check\n raise AuthenticationFailure, "No such session: %s" % e\nAuthenticationFailure: No such session: No row was found for one()\n', 'type': 2, 'ts': None, 'value': None} + + return result['value'] + +def execute_query(request, query): + if not 'manifold' in request.session or not 'auth' in request.session['manifold']: + request.session.flush() + raise Exception, "User not authenticated" + manifold_api_session_auth = request.session['manifold']['auth'] + return _execute_query(request, query, manifold_api_session_auth) + +def execute_admin_query(request, query): + admin_user, admin_password = ConfigEngine().manifold_admin_user_password() + admin_auth = {'AuthMethod': 'password', 'Username': admin_user, 'AuthString': admin_password} + return _execute_query(request, query, admin_auth) diff --git a/manifold/manifoldproxy.py b/manifold/manifoldproxy.py new file mode 100644 index 00000000..d0a8a3ea --- /dev/null +++ b/manifold/manifoldproxy.py @@ -0,0 +1,95 @@ +import json +import os.path + +# this is for django objects only +#from django.core import serializers +from django.http import HttpResponse, HttpResponseForbidden + +#from manifold.manifoldquery import ManifoldQuery +from manifold.core.query import Query +from manifold.core.result_value import ResultValue +from manifold.manifoldapi import ManifoldAPI +from manifold.manifoldresult import ManifoldException +from manifold.util.log import Log +from myslice.configengine import ConfigEngine + +debug=False +#debug=True + +# pretend the server only returns - empty lists to 'get' requests - this is to mimick +# misconfigurations or expired credentials or similar corner case situations +debug_empty=False +#debug_empty=True + +# this view is what the javascript talks to when it sends a query +# see also +# myslice/urls.py +# as well as +# static/js/manifold.js +def proxy (request,format): + """the view associated with /manifold/proxy/ +with the query passed using POST""" + + # expecting a POST + if request.method != 'POST': + print "manifoldproxy.api: unexpected method %s -- exiting"%request.method + return + # we only support json for now + # if needed in the future we should probably cater for + # format_in : how is the query encoded in POST + # format_out: how to serve the results + if format != 'json': + print "manifoldproxy.proxy: unexpected format %s -- exiting"%format + return + try: + # translate incoming POST request into a query object + if debug: print 'manifoldproxy.proxy: request.POST',request.POST + manifold_query = Query() + #manifold_query = ManifoldQuery() + manifold_query.fill_from_POST(request.POST) + # retrieve session for request + + # We allow some requests to use the ADMIN user account + if (manifold_query.get_from() == 'local:user' and manifold_query.get_action() == 'create') \ + or (manifold_query.get_from() == 'local:platform' and manifold_query.get_action() == 'get'): + admin_user, admin_password = ConfigEngine().manifold_admin_user_password() + manifold_api_session_auth = {'AuthMethod': 'password', 'Username': admin_user, 'AuthString': admin_password} + else: + print request.session['manifold'] + manifold_api_session_auth = request.session['manifold']['auth'] + + if debug_empty and manifold_query.action.lower()=='get': + json_answer=json.dumps({'code':0,'value':[]}) + print "By-passing : debug_empty & 'get' request : returning a fake empty list" + return HttpResponse (json_answer, mimetype="application/json") + + # actually forward + manifold_api= ManifoldAPI(auth=manifold_api_session_auth) + if debug: print '===> manifoldproxy.proxy: sending to backend', manifold_query + # for the benefit of the python code, manifoldAPI raises an exception if something is wrong + # however in this case we want to propagate the complete manifold result to the js world + + result = manifold_api.forward(manifold_query.to_dict()) + + # XXX TEMP HACK + if 'description' in result and result['description'] \ + and isinstance(result['description'], (tuple, list, set, frozenset)): + result [ 'description' ] = [ ResultValue.to_html (x) for x in result['description'] ] + + json_answer=json.dumps(result) + + return HttpResponse (json_answer, mimetype="application/json") + + except Exception,e: + print "** PROXY ERROR **",e + import traceback + traceback.print_exc() + +#################### +# see CSRF_FAILURE_VIEW in settings.py +# the purpose of redefining this was to display the failure reason somehow +# this however turns out disappointing/not very informative +failure_answer=[ "csrf_failure" ] +def csrf_failure(request, reason=""): + print "CSRF failure with reason '%s'"%reason + return HttpResponseForbidden (json.dumps (failure_answer), mimetype="application/json") diff --git a/manifold/manifoldresult.py b/manifold/manifoldresult.py new file mode 100644 index 00000000..4ffe072b --- /dev/null +++ b/manifold/manifoldresult.py @@ -0,0 +1,61 @@ +def enum(*sequential, **named): + enums = dict(zip(sequential, range(len(sequential))), **named) + return type('Enum', (), enums) + +ManifoldCode = enum ( + UNKNOWN_ERROR=-1, + SUCCESS=0, + SESSION_EXPIRED=1, + NOT_IMPLEMENTED=2, + SERVER_UNREACHABLE=3, +) + +_messages_ = { -1 : "Unknown", 0: "OK", 1: "Session Expired", 2: "Not Implemented", 3: "Backend server unreachable"} + +# being a dict this can be used with json.dumps +class ManifoldResult (dict): + def __init__ (self, code=ManifoldCode.SUCCESS, value=None, output=""): + self['code']=code + self['value']=value + self['output']=output + self['description'] = '' # Jordan: needed by javascript code + + def from_json (self, json_string): + d=json.dumps(json_string) + for k in ['code','value','output']: + self[k]=d[k] + + # raw accessors + def code (self): return self['code'] + def output (self): return self['output'] + + # this returns None if there's a problem, the value otherwise + def ok_value (self): + if self['code']==ManifoldCode.SUCCESS: + return self['value'] + + # both data in a single string + def error (self): + return "code=%s -- %s"%(self['code'],self['output']) + + + def __repr__ (self): + code=self['code'] + result="[MFresult %s (code=%s)"%(_messages_.get(code,"???"),code) + if code==0: + value=self['value'] + if isinstance(value,list): result += " [value=list with %d elts]"%len(value) + elif isinstance(value,dict): result += " [value=dict with keys %s]"%value.keys() + else: result += " [value=%s: %s]"%(type(value).__name__,value) + else: + result += " [output=%s]"%self['output'] + result += "]" + return result + +# probably simpler to use a single class and transport the whole result there +# instead of a clumsy set of derived classes +class ManifoldException (Exception): + def __init__ (self, manifold_result): + self.manifold_result=manifold_result + def __repr__ (self): + return "Manifold Exception %s"%(self.manifold_result.error()) diff --git a/manifold/metadata.py b/manifold/metadata.py new file mode 100644 index 00000000..de9bf2eb --- /dev/null +++ b/manifold/metadata.py @@ -0,0 +1,62 @@ +import json +import os.path + +from manifold.manifoldresult import ManifoldResult +from manifold.manifoldapi import ManifoldAPI + +from django.contrib import messages + +debug=False +#debug=True + +class MetaData: + + def __init__ (self, auth): + self.auth=auth + self.hash_by_object={} + + def fetch (self, request): + manifold_api = ManifoldAPI(self.auth) + fields = ['table', 'column.name', 'column.qualifier', 'column.type', + 'column.is_array', 'column.description', 'column.default', 'key', 'capability'] + #fields = ['table', 'column.column', + # 'column.description','column.header', 'column.title', + # 'column.unit', 'column.info_type', + # 'column.resource_type', 'column.value_type', + # 'column.allowed_values', 'column.platforms.platform', + # 'column.platforms.platform_url'] + request={ 'action': 'get', + 'object': 'local:object', # proposed to replace metadata:table + 'fields': fields , + } + result = manifold_api.forward(request) + + # xxx need a way to export error messages to the UI + if result['code'] == 1: # warning + # messages.warning(request, result['description']) + print ("METADATA WARNING -",request,result['description']) + elif result['code'] == 2: + # messages.error(request, result['description']) + print ("METADATA ERROR -",request,result['description']) + # XXX FAIL HERE XXX + return + + rows = result.ok_value() +# API errors will be handled by the outer logic +# if not rows: +# print "Failed to retrieve metadata",rows_result.error() +# rows=[] + self.hash_by_object = dict ( [ (row['table'], row) for row in rows ] ) + + def to_json(self): + return json.dumps(self.hash_by_object) + + def details_by_object (self, object): + return self.hash_by_object[object] + + def sorted_fields_by_object (self, object): + return self.hash_by_object[object]['column'].sort() + + def get_field_type(self, object, field): + if debug: print "Temp fix for metadata::get_field_type() -> consider moving to manifold.core.metadata soon" + return field diff --git a/manifold/static/css/manifold.css b/manifold/static/css/manifold.css new file mode 100644 index 00000000..42c034d3 --- /dev/null +++ b/manifold/static/css/manifold.css @@ -0,0 +1,4 @@ +.template { + visibility: hidden; + position: absolute; +} diff --git a/manifold/static/js/buffer.js b/manifold/static/js/buffer.js new file mode 100644 index 00000000..c33f34b8 --- /dev/null +++ b/manifold/static/js/buffer.js @@ -0,0 +1,40 @@ +/* Buffered DOM updates */ +var Buffer = Class.extend({ + + init: function(callback, callback_this) { + this._callback = callback; + this._callback_this = callback_this; + this._timerid = null; + this._num_elements = 0; + this._elements = Array(); + + this._interval = 1000; + + return this; + }, + + add: function(element) + { + this._elements.push(element); + if (this._num_elements == 0) { + this._timerid = setInterval( + (function(self) { //Self-executing func which takes 'this' as self + return function() { //Return a function in the context of 'self' + messages.debug("running callback"); + clearInterval(self._timerid); + self._callback.apply(self._callback_this); + } + })(this), + this._interval); + } + this._num_elements++; + }, + + get: function() { + var elements = this._elements; + this._elements = Array(); + this._num_elements = 0; + return elements; + }, + +}); diff --git a/manifold/static/js/class.js b/manifold/static/js/class.js new file mode 100644 index 00000000..63b570c3 --- /dev/null +++ b/manifold/static/js/class.js @@ -0,0 +1,64 @@ +/* Simple JavaScript Inheritance + * By John Resig http://ejohn.org/ + * MIT Licensed. + */ +// Inspired by base2 and Prototype +(function(){ + var initializing = false, fnTest = /xyz/.test(function(){xyz;}) ? /\b_super\b/ : /.*/; + + // The base Class implementation (does nothing) + this.Class = function(){}; + + // Create a new Class that inherits from this class + Class.extend = function(prop) { + var _super = this.prototype; + + // Instantiate a base class (but only create the instance, + // don't run the init constructor) + initializing = true; + var prototype = new this(); + initializing = false; + + // Copy the properties over onto the new prototype + for (var name in prop) { + // Check if we're overwriting an existing function + prototype[name] = typeof prop[name] == "function" && + typeof _super[name] == "function" && fnTest.test(prop[name]) ? + (function(name, fn){ + return function() { + var tmp = this._super; + + // Add a new ._super() method that is the same method + // but on the super-class + this._super = _super[name]; + + // The method only need to be bound temporarily, so we + // remove it when we're done executing + var ret = fn.apply(this, arguments); + this._super = tmp; + + return ret; + }; + })(name, prop[name]) : + prop[name]; + } + + // The dummy class constructor + function Class() { + // All construction is actually done in the init method + if ( !initializing && this.init ) + this.init.apply(this, arguments); + } + + // Populate our constructed prototype object + Class.prototype = prototype; + + // Enforce the constructor to be what we expect + Class.prototype.constructor = Class; + + // And make this class extendable + Class.extend = arguments.callee; + + return Class; + }; +})(); diff --git a/manifold/static/js/manifold-query.js b/manifold/static/js/manifold-query.js new file mode 100644 index 00000000..3116f84d --- /dev/null +++ b/manifold/static/js/manifold-query.js @@ -0,0 +1,271 @@ +function ManifoldQuery(action, object, timestamp, filters, params, fields, unique, query_uuid, aq, sq) { + // get, update, delete, create + var action; + // slice, user, network... + var object; + // timestamp, now, latest(cache) : date of the results queried + var timestamp; + // key(field),op(=<>),value + var filters; + // todo + var params; + // hostname, ip,... + var fields; + // 0,1 : list of element of an object or single object + var unique; + // query_uuid : unique identifier of a query + var query_uuid; + // Query : root query (no sub-Query) + var analyzed_query; + // {} : Assoc Table of sub-queries ["resources"->subQ1, "users"->subQ2] + var subqueries; + +/*------------------------------------------------------------- + Query properties are SQL like : +--------------------------------------------------------------- +SELECT fields FROM object WHERE filter; +UPDATE object SET field=value WHERE filter; / returns SELECT +DELETE FROM object WHERE filter +INSERT INTO object VALUES(field=value) +-------------------------------------------------------------*/ + + this.__repr = function () { + res = "ManifoldQuery "; +// res += " id=" + this.query_uuid; + res += " a=" + this.action; + res += " o=" + this.object; + res += " ts=" + this.timestamp; + res += " flts=" + this.filters; + res += " flds=" + this.fields; + res += " prms=" + this.params; + return res; + } + + this.clone = function() { + // + var q = new ManifoldQuery(); + q.action = this.action; + q.object = this.object; + q.timestamp = this.timestamp; + q.filters = this.filters.slice(); + q.fields = this.fields.slice(); + q.query_uuid = this.query_uuid; + + if (this.analyzed_query) + q.analyzed_query = this.analyzed_query.clone(); + else + q.analyzed_query = null; + + if (this.subqueries) { + q.subqueries = {} + for (method in this.subqueries) + q.subqueries[method] = this.subqueries[method].clone(); + } + + // deep extend not working for custom objects + // $.extend(true, q, this); + return q; + } + + this.add_filter = function(key, op, value) { + this.filters.push(new Array(key, op, value)); + } + this.update_filter = function(key, op, value) { + // Need to be improved... + // remove all occurrences of key if operation is not defined + if(!op){ + this.filters = jQuery.grep(this.filters, function(val, i) { + return val[0] != key; + }); + // Else remove the key+op filters + }else{ + this.filters = jQuery.grep(this.filters, function(val, i) {return (val[0] != key || val[1] != op);}); + } + this.filters.push(new Array(key, op, value)); + } + + this.remove_filter = function (key,op,value) { + // if operator is null then remove all occurences of this key + if(!op){ + this.filters = jQuery.grep(this.filters, function(val, i) { + return val[0] != key; + }); + }else{ + this.filters = jQuery.grep(this.filters, function(val, i) {return (val[0] != key || val[1] != op);}); + } + } + + // FIXME These functions computing diff's between queries are meant to be shared + this.diff_fields = function(otherQuery) { + var f1 = this.fields; + var f2 = otherQuery.fields; + + /* added elements are the ones in f2 not in f1 */ + var added = jQuery.grep(f2, function (x) { return jQuery.inArray(x, f1) == -1 }); + /* removed elements are the ones in f1 not in f2 */ + var removed = jQuery.grep(f1, function (x) { return jQuery.inArray(x, f2) == -1 }); + + return {'added':added, 'removed':removed}; + } + + // FIXME Modify filter to filters + this.diff_filter = function(otherQuery) { + var f1 = this.filters; + var f2 = otherQuery.filters; + + /* added elements are the ones in f2 not in f1 */ + var added = jQuery.grep(f2, function (x) { return !arrayInArray(x, f1)}); + /* removed elements are the ones in f1 not in f2 */ + var removed = jQuery.grep(f1, function (x) { return !arrayInArray(x, f2)}); + + return {'added':added, 'removed':removed}; + } + + // Callaback received 3 parameters: query, data, parent_query + this.iter_subqueries = function(callback, data) + { + rec = function(query, callback, data, parent_query) { + callback(query, data, parent_query); + jQuery.each(query.subqueries, function(object, subquery) { + rec(subquery, callback, data, query); + }); + }; + + if (this.analyzed_query !== undefined) + query = this.analyzed_query; + else + query = this; + + rec(query, callback, data, null); + } + + this.select = function(field) + { + this.fields.push(field); + } + + this.unselect = function(field) + { + this.fields = $.grep(this.fields, function(x) { return x != field; }); + } + +// we send queries as a json string now +// this.as_POST = function() { +// return {'action': this.action, 'object': this.object, 'timestamp': this.timestamp, +// 'filters': this.filters, 'params': this.params, 'fields': this.fields}; +// } + this.analyze_subqueries = function() { + /* adapted from the PHP function in com_tophat/includes/query.php */ + var q = new ManifoldQuery(); + q.query_uuid = this.query_uuid; + q.action = this.action; + q.object = this.object; + q.timestamp = this.timestamp; + + /* Filters */ + jQuery.each(this.filters, function(i, filter) { + var k = filter[0]; + var op = filter[1]; + var v = filter[2]; + var pos = k.indexOf('.'); + if (pos != -1) { + var object = k.substr(0, pos); + var field = k.substr(pos+1); + if (!q.subqueries[object]) { + q.subqueries[object] = new ManifoldQuery(); + q.subqueries[object].action = q.action; + q.subqueries[object].object = object; + q.subqueries[object].timestamp = q.timestamp; + } + q.subqueries[object].filters.push(Array(field, op, v)); + } else { + q.filters.push(filter); + } + }); + + /* Params */ + jQuery.each(this.params, function(param, value) { + var pos = param.indexOf('.'); + if (pos != -1) { + var object = param.substr(0, pos); + var field = param.substr(pos+1); + if (!q.subqueries[object]) { + q.subqueries[object] = new ManifoldQuery(); + q.subqueries[object].action = q.action; + q.subqueries[object].object = object; + q.subqueries[object].timestamp = q.timestamp; + } + q.subqueries[object].params[field] = value; + } else { + q.params[field] = value; + } + }); + + /* Fields */ + jQuery.each(this.fields, function(i, v) { + var pos = v.indexOf('.'); + if (pos != -1) { + var object = v.substr(0, pos); + var field = v.substr(pos+1); + if (!q.subqueries[object]) { + q.subqueries[object] = new ManifoldQuery(); + q.subqueries[object].action = q.action; + q.subqueries[object].object = object; + q.subqueries[object].timestamp = q.timestamp; + } + q.subqueries[object].fields.push(field); + } else { + q.fields.push(v); + } + }); + this.analyzed_query = q; + } + + /* constructor */ + if (typeof action == "undefined") + this.action = "get"; + else + this.action = action; + + if (typeof object == "undefined") + this.object = null; + else + this.object = object; + + if (typeof timestamp == "undefined") + this.timestamp = "now"; + else + this.timestamp = timestamp; + + if (typeof filters == "undefined") + this.filters = []; + else + this.filters = filters; + + if (typeof params == "undefined") + this.params = {}; + else + this.params = params; + + if (typeof fields == "undefined") + this.fields = []; + else + this.fields = fields; + + if (typeof unique == "undefined") + this.unique = false; + else + this.unique = unique; + + this.query_uuid = query_uuid; + + if (typeof aq == "undefined") + this.analyzed_query = null; + else + this.analyzed_query = aq; + + if (typeof sq == "undefined") + this.subqueries = {}; + else + this.subqueries = sq; +} diff --git a/manifold/static/js/manifold.js b/manifold/static/js/manifold.js new file mode 100644 index 00000000..3d526a67 --- /dev/null +++ b/manifold/static/js/manifold.js @@ -0,0 +1,1044 @@ +// utilities +function debug_dict_keys (msg, o) { + var keys=[]; + for (var k in o) keys.push(k); + messages.debug ("debug_dict_keys: " + msg + " keys= " + keys); +} +function debug_dict (msg, o) { + for (var k in o) messages.debug ("debug_dict: " + msg + " [" + k + "]=" + o[k]); +} +function debug_value (msg, value) { + messages.debug ("debug_value: " + msg + " " + value); +} +function debug_query (msg, query) { + if (query === undefined) messages.debug ("debug_query: " + msg + " -> undefined"); + else if (query == null) messages.debug ("debug_query: " + msg + " -> null"); + else if ('query_uuid' in query) messages.debug ("debug_query: " + msg + query.__repr()); + else messages.debug ("debug_query: " + msg + " query= " + query); +} + +// http://javascriptweblog.wordpress.com/2011/08/08/fixing-the-javascript-typeof-operator/ +Object.toType = (function toType(global) { + return function(obj) { + if (obj === global) { + return "global"; + } + return ({}).toString.call(obj).match(/\s([a-z|A-Z]+)/)[1].toLowerCase(); + } +})(this); + +/* ------------------------------------------------------------ */ + +// Constants that should be somehow moved to a plugin.js file +var FILTER_ADDED = 1; +var FILTER_REMOVED = 2; +var CLEAR_FILTERS = 3; +var FIELD_ADDED = 4; +var FIELD_REMOVED = 5; +var CLEAR_FIELDS = 6; +var NEW_RECORD = 7; +var CLEAR_RECORDS = 8; +var FIELD_STATE_CHANGED = 9; + +var IN_PROGRESS = 101; +var DONE = 102; + +/* Update requests related to subqueries */ +var SET_ADD = 201; +var SET_REMOVED = 202; + +// request +var FIELD_REQUEST_CHANGE = 301; +var FIELD_REQUEST_ADD = 302; +var FIELD_REQUEST_REMOVE = 303; +var FIELD_REQUEST_ADD_RESET = 304; +var FIELD_REQUEST_REMOVE_RESET = 305; +// status +var FIELD_REQUEST_PENDING = 401; +var FIELD_REQUEST_SUCCESS = 402; +var FIELD_REQUEST_FAILURE = 403; + +/* Query status */ +var STATUS_NONE = 500; // Query has not been started yet +var STATUS_GET_IN_PROGRESS = 501; // Query has been sent, no result has been received +var STATUS_GET_RECEIVED = 502; // Success +var STATUS_GET_ERROR = 503; // Error +var STATUS_UPDATE_PENDING = 504; +var STATUS_UPDATE_IN_PROGRESS = 505; +var STATUS_UPDATE_RECEIVED = 506; +var STATUS_UPDATE_ERROR = 507; + +/* Requests for query cycle */ +var RUN_UPDATE = 601; + +/* MANIFOLD types */ +var TYPE_VALUE = 1; +var TYPE_RECORD = 2; +var TYPE_LIST_OF_VALUES = 3; +var TYPE_LIST_OF_RECORDS = 4; + + +// A structure for storing queries + +function QueryExt(query, parent_query_ext, main_query_ext, update_query_ext, disabled) { + + /* Constructor */ + if (typeof query == "undefined") + throw "Must pass a query in QueryExt constructor"; + this.query = query; + this.parent_query_ext = (typeof parent_query_ext == "undefined") ? null : parent_query_ext; + this.main_query_ext = (typeof main_query_ext == "undefined") ? null : main_query_ext; + this.update_query_ext = (typeof update_query_ext == "undefined") ? null : update_query_ext; + this.update_query_orig_ext = (typeof update_query_orig_ext == "undefined") ? null : update_query_orig_ext; + this.disabled = (typeof update_query_ext == "undefined") ? false : disabled; + + this.status = null; + this.results = null; + // update_query null unless we are a main_query (aka parent_query == null); only main_query_fields can be updated... +} + +function QueryStore() { + + this.main_queries = {}; + this.analyzed_queries = {}; + + /* Insertion */ + + this.insert = function(query) { + // We expect only main_queries are inserted + + /* If the query has not been analyzed, then we analyze it */ + if (query.analyzed_query == null) { + query.analyze_subqueries(); + } + + /* We prepare the update query corresponding to the main query and store both */ + /* Note: they have the same UUID */ + + // XXX query.change_action() should become deprecated + update_query = query.clone(); + update_query.action = 'update'; + update_query.analyzed_query.action = 'update'; + update_query.params = {}; + update_query_ext = new QueryExt(update_query); + + /* We remember the original query to be able to reset it */ + update_query_orig_ext = new QueryExt(update_query.clone()); + + + /* We store the main query */ + query_ext = new QueryExt(query, null, null, update_query_ext, update_query_orig_ext, false); + manifold.query_store.main_queries[query.query_uuid] = query_ext; + /* Note: the update query does not have an entry! */ + + + // The query is disabled; since it is incomplete until we know the content of the set of subqueries + // XXX unless we have no subqueries ??? + // we will complete with params when records are received... this has to be done by the manager + // SET_ADD, SET_REMOVE will change the status of the elements of the set + // UPDATE will change also, etc. + // XXX We need a proper structure to store this information... + + // We also need to insert all queries and subqueries from the analyzed_query + // XXX We need the root of all subqueries + query.iter_subqueries(function(sq, data, parent_query) { + if (parent_query) + parent_query_ext = manifold.query_store.find_analyzed_query_ext(parent_query.query_uuid); + else + parent_query_ext = null; + // XXX parent_query_ext == false + // XXX main.subqueries = {} # Normal, we need analyzed_query + sq_ext = new QueryExt(sq, parent_query_ext, query_ext) + manifold.query_store.analyzed_queries[sq.query_uuid] = sq_ext; + }); + + // XXX We have spurious update queries... + } + + /* Searching */ + + this.find_query_ext = function(query_uuid) { + return this.main_queries[query_uuid]; + } + + this.find_query = function(query_uuid) { + return this.find_query_ext(query_uuid).query; + } + + this.find_analyzed_query_ext = function(query_uuid) { + return this.analyzed_queries[query_uuid]; + } + + this.find_analyzed_query = function(query_uuid) { + return this.find_analyzed_query_ext(query_uuid).query; + } +} + +/*! + * This namespace holds functions for globally managing query objects + * \Class Manifold + */ +var manifold = { + + /************************************************************************** + * Helper functions + **************************************************************************/ + + separator: '__', + + get_type: function(variable) { + switch(Object.toType(variable)) { + case 'number': + case 'string': + return TYPE_VALUE; + case 'object': + return TYPE_RECORD; + case 'array': + if ((variable.length > 0) && (Object.toType(variable[0]) === 'object')) + return TYPE_LIST_OF_RECORDS; + else + return TYPE_LIST_OF_VALUES; + } + }, + + /************************************************************************** + * Metadata management + **************************************************************************/ + + metadata: { + + get_table: function(method) { + var table = MANIFOLD_METADATA[method]; + return (typeof table === 'undefined') ? null : table; + }, + + get_columns: function(method) { + var table = this.get_table(method); + if (!table) { + return null; + } + + return (typeof table.column === 'undefined') ? null : table.column; + }, + + get_key: function(method) { + var table = this.get_table(method); + if (!table) + return null; + + return (typeof table.key === 'undefined') ? null : table.key; + }, + + + get_column: function(method, name) { + var columns = this.get_columns(method); + if (!columns) + return null; + + $.each(columns, function(i, c) { + if (c.name == name) + return c + }); + return null; + }, + + get_type: function(method, name) { + var table = this.get_table(method); + if (!table) + return null; + + return (typeof table.type === 'undefined') ? null : table.type; + } + + }, + + /************************************************************************** + * Query management + **************************************************************************/ + + query_store: new QueryStore(), + + // XXX Remaining functions are deprecated since they are replaced by the query store + + /*! + * Associative array storing the set of queries active on the page + * \memberof Manifold + */ + all_queries: {}, + + /*! + * Insert a query in the global hash table associating uuids to queries. + * If the query has no been analyzed yet, let's do it. + * \fn insert_query(query) + * \memberof Manifold + * \param ManifoldQuery query Query to be added + */ + insert_query : function (query) { + // NEW API + manifold.query_store.insert(query); + + // FORMER API + if (query.analyzed_query == null) { + query.analyze_subqueries(); + } + manifold.all_queries[query.query_uuid]=query; + }, + + /*! + * Returns the query associated to a UUID + * \fn find_query(query_uuid) + * \memberof Manifold + * \param string query_uuid The UUID of the query to be returned + */ + find_query : function (query_uuid) { + return manifold.all_queries[query_uuid]; + }, + + /************************************************************************** + * Query execution + **************************************************************************/ + + // trigger a query asynchroneously + proxy_url : '/manifold/proxy/json/', + + // reasonably low-noise, shows manifold requests coming in and out + asynchroneous_debug : true, + // print our more details on result publication and related callbacks + pubsub_debug : false, + + /** + * \brief We use js function closure to be able to pass the query (array) + * to the callback function used when data is received + */ + success_closure: function(query, publish_uuid, callback) { + return function(data, textStatus) { + manifold.asynchroneous_success(data, query, publish_uuid, callback); + } + }, + + run_query: function(query, callback) { + // default value for callback = null + if (typeof callback === 'undefined') + callback = null; + + var query_json = JSON.stringify(query); + + /* Nothing related to pubsub here... for the moment at least. */ + //query.iter_subqueries(function (sq) { + // manifold.raise_record_event(sq.query_uuid, IN_PROGRESS); + //}); + + $.post(manifold.proxy_url, {'json': query_json} , manifold.success_closure(query, null, callback)); + }, + + // Executes all async. queries - intended for the javascript header to initialize queries + // input queries are specified as a list of {'query_uuid': } + // each plugin is responsible for managing its spinner through on_query_in_progress + asynchroneous_exec : function (query_exec_tuples) { + + // Loop through input array, and use publish_uuid to publish back results + $.each(query_exec_tuples, function(index, tuple) { + var query=manifold.find_query(tuple.query_uuid); + var query_json=JSON.stringify (query); + var publish_uuid=tuple.publish_uuid; + // by default we publish using the same uuid of course + if (publish_uuid==undefined) publish_uuid=query.query_uuid; + if (manifold.pubsub_debug) { + messages.debug("sending POST on " + manifold.proxy_url + query.__repr()); + } + + query.iter_subqueries(function (sq) { + manifold.raise_record_event(sq.query_uuid, IN_PROGRESS); + }); + + // not quite sure what happens if we send a string directly, as POST data is named.. + // this gets reconstructed on the proxy side with ManifoldQuery.fill_from_POST + $.post(manifold.proxy_url, {'json':query_json}, + manifold.success_closure(query, publish_uuid, tuple.callback)); + }) + }, + + /** + * \brief Forward a query to the manifold backend + * \param query (dict) the query to be executed asynchronously + * \param callback (function) the function to be called when the query terminates + */ + forward: function(query, callback) { + var query_json = JSON.stringify(query); + $.post(manifold.proxy_url, {'json': query_json} , + manifold.success_closure(query, query.query_uuid, callback)); + }, + + /*! + * Returns whether a query expects a unique results. + * This is the case when the filters contain a key of the object + * \fn query_expects_unique_result(query) + * \memberof Manifold + * \param ManifoldQuery query Query for which we are testing whether it expects a unique result + */ + query_expects_unique_result: function(query) { + /* XXX we need functions to query metadata */ + //var keys = MANIFOLD_METADATA[query.object]['keys']; /* array of array of field names */ + /* TODO requires keys in metadata */ + return true; + }, + + /*! + * Publish result + * \fn publish_result(query, results) + * \memberof Manifold + * \param ManifoldQuery query Query which has received results + * \param array results results corresponding to query + */ + publish_result: function(query, result) { + if (typeof result === 'undefined') + result = []; + + // NEW PLUGIN API + manifold.raise_record_event(query.query_uuid, CLEAR_RECORDS); + if (manifold.pubsub_debug) + messages.debug(".. publish_result (1) "); + var count=0; + $.each(result, function(i, record) { + manifold.raise_record_event(query.query_uuid, NEW_RECORD, record); + count += 1; + }); + if (manifold.pubsub_debug) + messages.debug(".. publish_result (2) has used NEW API on " + count + " records"); + manifold.raise_record_event(query.query_uuid, DONE); + if (manifold.pubsub_debug) + messages.debug(".. publish_result (3) has used NEW API to say DONE"); + + // OLD PLUGIN API BELOW + /* Publish an update announce */ + var channel="/results/" + query.query_uuid + "/changed"; + if (manifold.pubsub_debug) + messages.debug(".. publish_result (4) OLD API on channel" + channel); + + $.publish(channel, [result, query]); + + if (manifold.pubsub_debug) + messages.debug(".. publish_result (5) END q=" + query.__repr()); + }, + + /*! + * Recursively publish result + * \fn publish_result_rec(query, result) + * \memberof Manifold + * \param ManifoldQuery query Query which has received result + * \param array result result corresponding to query + * + * Note: this function works on the analyzed query + */ + publish_result_rec: function(query, result) { + /* If the result is not unique, only publish the top query; + * otherwise, publish the main object as well as subqueries + * XXX how much recursive are we ? + */ + if (manifold.pubsub_debug) + messages.debug (">>>>> publish_result_rec " + query.object); + if (manifold.query_expects_unique_result(query)) { + /* Also publish subqueries */ + $.each(query.subqueries, function(object, subquery) { + manifold.publish_result_rec(subquery, result[0][object]); + /* TODO remove object from result */ + }); + } + if (manifold.pubsub_debug) + messages.debug ("===== publish_result_rec " + query.object); + + manifold.publish_result(query, result); + + if (manifold.pubsub_debug) + messages.debug ("<<<<< publish_result_rec " + query.object); + }, + + setup_update_query: function(query, records) { + // We don't prepare an update query if the result has more than 1 entry + if (records.length != 1) + return; + var query_ext = manifold.query_store.find_query_ext(query.query_uuid); + + var record = records[0]; + + var update_query_ext = query_ext.update_query_ext; + var update_query = update_query_ext.query; + var update_query_ext = query_ext.update_query_ext; + var update_query_orig = query_ext.update_query_orig_ext.query; + + // Testing whether the result has subqueries (one level deep only) + // iif the query has subqueries + var count = 0; + var obj = query.analyzed_query.subqueries; + for (method in obj) { + if (obj.hasOwnProperty(method)) { + var key = manifold.metadata.get_key(method); + if (!key) + continue; + if (key.length > 1) + continue; + key = key[0]; + var sq_keys = []; + var subrecords = record[method]; + if (!subrecords) + continue + $.each(subrecords, function (i, subrecord) { + sq_keys.push(subrecord[key]); + }); + update_query.params[method] = sq_keys; + update_query_orig.params[method] = sq_keys.slice(); + count++; + } + } + + if (count > 0) { + update_query_ext.disabled = false; + update_query_orig_ext.disabled = false; + } + }, + + process_get_query_records: function(query, records) { + this.setup_update_query(query, records); + + /* Publish full results */ + var tmp_query = manifold.find_query(query.query_uuid); + manifold.publish_result_rec(tmp_query.analyzed_query, records); + }, + + /** + * + * What we need to do when receiving results from an update query: + * - differences between what we had, what we requested, and what we obtained + * . what we had : update_query_orig (simple fields and set fields managed differently) + * . what we requested : update_query + * . what we received : records + * - raise appropriate events + * + * The normal process is that results similar to Get will be pushed in the + * pubsub mechanism, thus repopulating everything while we only need + * diff's. This means we need to move the publish functionalities in the + * previous 'process_get_query_records' function. + */ + process_update_query_records: function(query, records) { + // First issue: we request everything, and not only what we modify, so will will have to ignore some fields + var query_uuid = query.query_uuid; + var query_ext = manifold.query_store.find_analyzed_query_ext(query_uuid); + var update_query = query_ext.main_query_ext.update_query_ext.query; + var update_query_orig = query_ext.main_query_ext.update_query_orig_ext.query; + + // Since we update objects one at a time, we can get the first record + var record = records[0]; + + // Let's iterate over the object properties + for (var field in record) { + switch (this.get_type(record[field])) { + case TYPE_VALUE: + // Did we ask for a change ? + var update_value = update_query[field]; + if (!update_value) + // Not requested, if it has changed: OUT OF SYNC + // How we can know ? + // We assume it won't have changed + continue; + + var result_value = record[field]; + if (!result_value) + throw "Internal error"; + + data = { + request: FIELD_REQUEST_CHANGE, + key : field, + value : update_value, + status: (update_value == result_value) ? FIELD_REQUEST_SUCCESS : FIELD_REQUEST_FAILURE, + } + manifold.raise_record_event(query_uuid, FIELD_STATE_CHANGED, data); + + break; + case TYPE_RECORD: + throw "Not implemented"; + break; + + case TYPE_LIST_OF_VALUES: + // Same as list of records, but we don't have to extract keys + var result_keys = record[field] + + // The rest of exactly the same (XXX factorize) + var update_keys = update_query_orig.params[field]; + var query_keys = update_query.params[field]; + var added_keys = $.grep(query_keys, function (x) { return $.inArray(x, update_keys) == -1 }); + var removed_keys = $.grep(update_keys, function (x) { return $.inArray(x, query_keys) == -1 }); + + + $.each(added_keys, function(i, key) { + if ($.inArray(key, result_keys) == -1) { + data = { + request: FIELD_REQUEST_ADD, + key : field, + value : key, + status: FIELD_REQUEST_FAILURE, + } + } else { + data = { + request: FIELD_REQUEST_ADD, + key : field, + value : key, + status: FIELD_REQUEST_SUCCESS, + } + } + manifold.raise_record_event(query_uuid, FIELD_STATE_CHANGED, data); + }); + $.each(removed_keys, function(i, key) { + if ($.inArray(key, result_keys) == -1) { + data = { + request: FIELD_REQUEST_REMOVE, + key : field, + value : key, + status: FIELD_REQUEST_SUCCESS, + } + } else { + data = { + request: FIELD_REQUEST_REMOVE, + key : field, + value : key, + status: FIELD_REQUEST_FAILURE, + } + } + manifold.raise_record_event(query_uuid, FIELD_STATE_CHANGED, data); + }); + + + break; + case TYPE_LIST_OF_RECORDS: + // example: slice.resource + // - update_query_orig.params.resource = resources in slice before update + // - update_query.params.resource = resource requested in slice + // - keys from field = resources obtained + var key = manifold.metadata.get_key(field); + if (!key) + continue; + if (key.length > 1) { + throw "Not implemented"; + continue; + } + key = key[0]; + + /* XXX should be modified for multiple keys */ + var result_keys = $.map(record[field], function(x) { return x[key]; }); + + var update_keys = update_query_orig.params[field]; + var query_keys = update_query.params[field]; + var added_keys = $.grep(query_keys, function (x) { return $.inArray(x, update_keys) == -1 }); + var removed_keys = $.grep(update_keys, function (x) { return $.inArray(x, query_keys) == -1 }); + + + $.each(added_keys, function(i, key) { + if ($.inArray(key, result_keys) == -1) { + data = { + request: FIELD_REQUEST_ADD, + key : field, + value : key, + status: FIELD_REQUEST_FAILURE, + } + } else { + data = { + request: FIELD_REQUEST_ADD, + key : field, + value : key, + status: FIELD_REQUEST_SUCCESS, + } + } + manifold.raise_record_event(query_uuid, FIELD_STATE_CHANGED, data); + }); + $.each(removed_keys, function(i, key) { + if ($.inArray(key, result_keys) == -1) { + data = { + request: FIELD_REQUEST_REMOVE, + key : field, + value : key, + status: FIELD_REQUEST_SUCCESS, + } + } else { + data = { + request: FIELD_REQUEST_REMOVE, + key : field, + value : key, + status: FIELD_REQUEST_FAILURE, + } + } + manifold.raise_record_event(query_uuid, FIELD_STATE_CHANGED, data); + }); + + + break; + } + } + + // XXX Now we need to adapt 'update' and 'update_orig' queries as if we had done a get + this.setup_update_query(query, records); + }, + + process_query_records: function(query, records) { + if (query.action == 'get') { + this.process_get_query_records(query, records); + } else if (query.action == 'update') { + this.process_update_query_records(query, records); + } + }, + + // if set callback is provided it is called + // most of the time publish_uuid will be query.query_uuid + // however in some cases we wish to publish the result under a different uuid + // e.g. an updater wants to publish its result as if from the original (get) query + asynchroneous_success : function (data, query, publish_uuid, callback) { + // xxx should have a nicer declaration of that enum in sync with the python code somehow + + var start = new Date(); + if (manifold.asynchroneous_debug) + messages.debug(">>>>>>>>>> asynchroneous_success query.object=" + query.object); + + if (data.code == 2) { // ERROR + // We need to make sense of error codes here + alert("Your session has expired, please log in again"); + window.location="/logout/"; + if (manifold.asynchroneous_debug) { + duration=new Date()-start; + messages.debug ("<<<<<<<<<< asynchroneous_success " + query.object + " -- error returned - logging out " + duration + " ms"); + } + return; + } + if (data.code == 1) { // WARNING + messages.error("Some errors have been received from the manifold backend at " + MANIFOLD_URL + " [" + data.description + "]"); + // publish error code and text message on a separate channel for whoever is interested + if (publish_uuid) + $.publish("/results/" + publish_uuid + "/failed", [data.code, data.description] ); + + } + + // If a callback has been specified, we redirect results to it + if (!!callback) { + callback(data); + if (manifold.asynchroneous_debug) { + duration=new Date()-start; + messages.debug ("<<<<<<<<<< asynchroneous_success " + query.object + " -- callback ended " + duration + " ms"); + } + return; + } + + if (manifold.asynchroneous_debug) + messages.debug ("========== asynchroneous_success " + query.object + " -- before process_query_records [" + query.query_uuid +"]"); + + // once everything is checked we can use the 'value' part of the manifoldresult + var result=data.value; + if (result) { + /* Eventually update the content of related queries (update, etc) */ + this.process_query_records(query, result); + + /* Publish results: disabled here, done in the previous call */ + //tmp_query = manifold.find_query(query.query_uuid); + //manifold.publish_result_rec(tmp_query.analyzed_query, result); + } + if (manifold.asynchroneous_debug) { + duration=new Date()-start; + messages.debug ("<<<<<<<<<< asynchroneous_success " + query.object + " -- done " + duration + " ms"); + } + + }, + + /************************************************************************** + * Plugin API helpers + **************************************************************************/ + + raise_event_handler: function(type, query_uuid, event_type, value) { + if (manifold.pubsub_debug) + messages.debug("raise_event_handler, quuid="+query_uuid+" type="+type+" event_type="+event_type); + if ((type != 'query') && (type != 'record')) + throw 'Incorrect type for manifold.raise_event()'; + // xxx we observe quite a lot of incoming calls with an undefined query_uuid + // this should be fixed upstream in manifold I expect + if (query_uuid === undefined) { + messages.warning("undefined query in raise_event_handler"); + return; + } + + // notify the change to objects that either listen to this channel specifically, + // or to the wildcard channel + var channels = [ manifold.get_channel(type, query_uuid), manifold.get_channel(type, '*') ]; + + $.each(channels, function(i, channel) { + if (value === undefined) { + if (manifold.pubsub_debug) messages.debug("triggering [no value] on channel="+channel+" and event_type="+event_type); + $('.pubsub').trigger(channel, [event_type]); + } else { + if (manifold.pubsub_debug) messages.debug("triggering [value="+value+"] on channel="+channel+" and event_type="+event_type); + $('.pubsub').trigger(channel, [event_type, value]); + } + }); + }, + + raise_query_event: function(query_uuid, event_type, value) { + manifold.raise_event_handler('query', query_uuid, event_type, value); + }, + + raise_record_event: function(query_uuid, event_type, value) { + manifold.raise_event_handler('record', query_uuid, event_type, value); + }, + + + raise_event: function(query_uuid, event_type, value) { + // Query uuid has been updated with the key of a new element + query_ext = manifold.query_store.find_analyzed_query_ext(query_uuid); + query = query_ext.query; + + switch(event_type) { + case FIELD_STATE_CHANGED: + // value is an object (request, key, value, status) + // update is only possible is the query is not pending, etc + // SET_ADD is on a subquery, FIELD_STATE_CHANGED on the query itself + // we should map SET_ADD on this... + + // 1. Update internal query store about the change in status + + // 2. Update the update query + update_query = query_ext.main_query_ext.update_query_ext.query; + update_query_orig = query_ext.main_query_ext.update_query_orig_ext.query; + + switch(value.request) { + case FIELD_REQUEST_CHANGE: + if (update_query.params[value.key] === undefined) + update_query.params[value.key] = Array(); + update_query.params[value.key] = value.value; + break; + case FIELD_REQUEST_ADD: + if ($.inArray(value.value, update_query_orig.params[value.key]) != -1) + value.request = FIELD_REQUEST_ADD_RESET; + if (update_query.params[value.key] === undefined) + update_query.params[value.key] = Array(); + update_query.params[value.key].push(value.value); + break; + case FIELD_REQUEST_REMOVE: + if ($.inArray(value.value, update_query_orig.params[value.key]) == -1) + value.request = FIELD_REQUEST_REMOVE_RESET; + + var arr = update_query.params[value.key]; + arr = $.grep(arr, function(x) { return x != value.value; }); + if (update_query.params[value.key] === undefined) + update_query.params[value.key] = Array(); + update_query.params[value.key] = arr; + + break; + case FIELD_REQUEST_ADD_RESET: + case FIELD_REQUEST_REMOVE_RESET: + // XXX We would need to keep track of the original query + throw "Not implemented"; + break; + } + + // 3. Inform others about the change + // a) the main query... + manifold.raise_record_event(query_uuid, event_type, value); + + // b) subqueries eventually (dot in the key) + // Let's unfold + var path_array = value.key.split('.'); + var value_key = value.key.split('.'); + + var cur_query = query; + if (cur_query.analyzed_query) + cur_query = cur_query.analyzed_query; + $.each(path_array, function(i, method) { + cur_query = cur_query.subqueries[method]; + value_key.shift(); // XXX check that method is indeed shifted + }); + value.key = value_key; + + manifold.raise_record_event(cur_query.query_uuid, event_type, value); + + // XXX make this DOT a global variable... could be '/' + break; + + case SET_ADD: + case SET_REMOVED: + + // update is only possible is the query is not pending, etc + // CHECK status ! + + // XXX we can only update subqueries of the main query. Check ! + // assert query_ext.parent_query == query_ext.main_query + // old // update_query = query_ext.main_query_ext.update_query_ext.query; + + // This SET_ADD is called on a subquery, so we have to + // recontruct the path of the key in the main_query + // We then call FIELD_STATE_CHANGED which is the equivalent for the main query + + var path = ""; + var sq = query_ext; + while (sq.parent_query_ext) { + if (path != "") + path = '.' + path; + path = sq.query.object + path; + sq = sq.parent_query_ext; + } + + main_query = query_ext.main_query_ext.query; + data = { + request: (event_type == SET_ADD) ? FIELD_REQUEST_ADD : FIELD_REQUEST_REMOVE, + key : path, + value : value, + status: FIELD_REQUEST_PENDING, + }; + this.raise_event(main_query.query_uuid, FIELD_STATE_CHANGED, data); + + // old //update_query.params[path].push(value); + // old // console.log('Updated query params', update_query); + // NOTE: update might modify the fields in Get + // NOTE : we have to modify all child queries + // NOTE : parts of a query might not be started (eg slice.measurements, how to handle ?) + + // if everything is done right, update_query should not be null. + // It is updated when we received results from the get query + // object = the same as get + // filter = key : update a single object for now + // fields = the same as get + manifold.raise_query_event(query_uuid, event_type, value); + + break; + + case RUN_UPDATE: + manifold.run_query(query_ext.main_query_ext.update_query_ext.query); + break; + + case FILTER_ADDED: +// Thierry - this is probably wrong but intended as a hotfix +// http://trac.myslice.info/ticket/32 +// manifold.raise_query_event(query_uuid, event_type, value); + break; + case FILTER_REMOVED: + manifold.raise_query_event(query_uuid, event_type, value); + break; + case FIELD_ADDED: + main_query = query_ext.main_query_ext.query; + main_update_query = query_ext.main_query_ext.update_query; + query.select(value); + + // Here we need the full path through all subqueries + path = "" + // XXX We might need the query name in the QueryExt structure + main_query.select(value); + + // XXX When is an update query associated ? + // XXX main_update_query.select(value); + + manifold.raise_query_event(query_uuid, event_type, value); + break; + + case FIELD_REMOVED: + query = query_ext.query; + main_query = query_ext.main_query_ext.query; + main_update_query = query_ext.main_query_ext.update_query; + query.unselect(value); + main_query.unselect(value); + + // We need to inform about changes in these queries to the respective plugins + // Note: query & main_query have the same UUID + manifold.raise_query_event(query_uuid, event_type, value); + break; + } + // We need to inform about changes in these queries to the respective plugins + // Note: query, main_query & update_query have the same UUID + manifold.raise_query_event(query_uuid, event_type, value); + // We are targeting the same object with get and update + // The notion of query is bad, we should have a notion of destination, and issue queries on the destination + // NOTE: Editing a subquery == editing a local view on the destination + + // XXX We might need to run the new query again and manage the plugins in the meantime with spinners... + // For the time being, we will collect all columns during the first query + }, + + /* Publish/subscribe channels for internal use */ + get_channel: function(type, query_uuid) { + if ((type !== 'query') && (type != 'record')) + return null; + return '/' + type + '/' + query_uuid; + }, + +}; // manifold object +/* ------------------------------------------------------------ */ + +(function($) { + + // OLD PLUGIN API: extend jQuery/$ with pubsub capabilities + // https://gist.github.com/661855 + var o = $({}); + $.subscribe = function( channel, selector, data, fn) { + /* borrowed from jQuery */ + if ( data == null && fn == null ) { + // ( channel, fn ) + fn = selector; + data = selector = undefined; + } else if ( fn == null ) { + if ( typeof selector === "string" ) { + // ( channel, selector, fn ) + fn = data; + data = undefined; + } else { + // ( channel, data, fn ) + fn = data; + data = selector; + selector = undefined; + } + } + /* */ + + /* We use an indirection function that will clone the object passed in + * parameter to the subscribe callback + * + * FIXME currently we only clone query objects which are the only ones + * supported and editable, we might have the same issue with results but + * the page load time will be severely affected... + */ + o.on.apply(o, [channel, selector, data, function() { + for(i = 1; i < arguments.length; i++) { + if ( arguments[i].constructor.name == 'ManifoldQuery' ) + arguments[i] = arguments[i].clone(); + } + fn.apply(o, arguments); + }]); + }; + + $.unsubscribe = function() { + o.off.apply(o, arguments); + }; + + $.publish = function() { + o.trigger.apply(o, arguments); + }; + +}(jQuery)); + +/* ------------------------------------------------------------ */ + +//http://stackoverflow.com/questions/5100539/django-csrf-check-failing-with-an-ajax-post-request +//make sure to expose csrf in our outcoming ajax/post requests +$.ajaxSetup({ + beforeSend: function(xhr, settings) { + function getCookie(name) { + var cookieValue = null; + if (document.cookie && document.cookie != '') { + var cookies = document.cookie.split(';'); + for (var i = 0; i < cookies.length; i++) { + var cookie = jQuery.trim(cookies[i]); + // Does this cookie string begin with the name we want? + if (cookie.substring(0, name.length + 1) == (name + '=')) { + cookieValue = decodeURIComponent(cookie.substring(name.length + 1)); + break; + } + } + } + return cookieValue; + } + if (!(/^http:.*/.test(settings.url) || /^https:.*/.test(settings.url))) { + // Only send the token to relative URLs i.e. locally. + xhr.setRequestHeader("X-CSRFToken", getCookie('csrftoken')); + } + } +}); diff --git a/manifold/static/js/metadata.js b/manifold/static/js/metadata.js new file mode 100644 index 00000000..28e6c12d --- /dev/null +++ b/manifold/static/js/metadata.js @@ -0,0 +1,53 @@ +// MANIFOLD_METADATA was formerly known as all_headers +var metadata = { + get : function () { + return MANIFOLD_METADATA; + }, + // returns all fields of a given object + fields : function (object) { + var result=new Array(); + jQuery.each(MANIFOLD_METADATA, function(s,obj){ + if(s==object){ + jQuery.each(obj['column'], function(i,f){ + result.push(f); + }); + return false; + } + }); + result.sort(sort_by('column', false, function(a){return a.toUpperCase()})); + //result=jQuery(result).sort("column", "asc"); + return result; + }, + // returns all properties of a given field + field : function (object, field) { + var result=new Array(); + jQuery.each(MANIFOLD_METADATA, function(s,obj){ + if(s==object){ + jQuery.each(obj['column'], function(i,f){ + if(f['column']==field){ + result.push(f); + return false; + } + }); + return false; + } + }); + return result[0]; + }, + // returns the value of a property from a field within a object (type of object : resource,node,lease,slice...) + property : function (object, field, property) { + var result=null; + jQuery.each(MANIFOLD_METADATA, function(s,obj){ + if(s==object){ + jQuery.each(obj['column'], function(i,f){ + if(f['column']==field){ + result=f[property]; + return false; + } + }); + return false; + } + }); + return result; + }, +} // metadata object diff --git a/manifold/static/js/plugin.js b/manifold/static/js/plugin.js new file mode 100644 index 00000000..24d41129 --- /dev/null +++ b/manifold/static/js/plugin.js @@ -0,0 +1,315 @@ +// INHERITANCE +// http://alexsexton.com/blog/2010/02/using-inheritance-patterns-to-organize-large-jquery-applications/ +// We will use John Resig's proposal + +// http://pastie.org/517177 + +// NOTE: missing a destroy function + +$.plugin = function(name, object) { + $.fn[name] = function(options) { + var args = Array.prototype.slice.call(arguments, 1); + return this.each(function() { + var instance = $.data(this, name); + if (instance) { + instance[options].apply(instance, args); + } else { + instance = $.data(this, name, new object(options, this)); + } + }); + }; +}; + +// set to either +// * false or undefined or none : no debug +// * true : trace all event calls +// * [ 'in_progress', 'query_done' ] : would only trace to these events +var plugin_debug=false; +plugin_debug = [ 'in_progress', 'query_done' ]; + +var Plugin = Class.extend({ + + init: function(options, element) { + // Mix in the passed in options with the default options + this.options = $.extend({}, this.default_options, options); + + // Save the element reference, both as a jQuery + // reference and a normal reference + this.element = element; + this.$element = $(element); + // programmatically add specific class for publishing events + // used in manifold.js for triggering API events + if ( ! this.$element.hasClass('pubsub')) this.$element.addClass('pubsub'); + + // return this so we can chain/use the bridge with less code. + return this; + }, + + has_query_handler: function() { + return (typeof this.on_filter_added === 'function'); + }, + + // do we need to log API calls ? + _is_in : function (obj, arr) { + for(var i=0; i + // and then $("#some-id-that-comes-from-the-db") + // however the syntax for that selector prevents from using some characters in id + // and so for some of our ids this won't work + // instead of 'flattening' we now do this instead + // + // and to retrieve it + // $("[some_id='then!we:can+use.what$we!want']") + // which thanks to the quotes, works; and you can use this with id as well in fact + // of course if now we have quotes in the id it's going to squeak, but well.. + + // escape (read: backslashes) some meta-chars in input + escape_id: function(id) { + if( id !== undefined){ + return id.replace( /(:|\.|\[|\])/g, "\\$1" ); + }else{ + return "undefined-id"; + } + }, + + id_from_record: function(method, record) { + var keys = manifold.metadata.get_key(method); + if (!keys) + return; + if (keys.length > 1) + return; + + var key = keys[0]; + switch (Object.toType(key)) { + case 'string': + if (!(key in record)) + return null; + return this.id_from_key(key, record[key]); + + default: + throw 'Not implemented'; + } + }, + + key_from_id: function(id) { + // NOTE this works only for simple keys + + var array; + if (typeof id === 'string') { + array = id.split(manifold.separator); + } else { // We suppose we have an array ('object') + array = id; + } + + // arguments has the initial id but lacks the key field name (see id_from_key), so we are even + // we finally add +1 for the plugin_uuid at the beginning + return array[arguments.length + 1]; + }, + + // TOGGLE + // plugin-helper.js is about managing toggled state + // it would be beneficial to merge it in here + toggle_on: function () { return this.toggle("true"); }, + toggle_off: function () { return this.toggle("false"); }, + toggle: function (status) { + plugin_helper.set_toggle_status (this.options.plugin_uuid,status); + }, + + /* SPIN */ + // use spin() to get our default spin settings (called presets) + // use spin(true) to get spin's builtin defaults + // you can also call spin_presets() yourself and tweak what you need to, like topmenuvalidation does + spin: function (presets) { + var presets = ( presets === undefined ) ? spin_presets() : presets; + try { this.$element.spin(presets); } + catch (err) { messages.debug("Cannot turn on spin " + err); } + }, + + unspin: function() { + try { this.$element.spin(false); } + catch (err) { messages.debug("Cannot turn off spin " + err); } + }, + + /* TEMPLATE */ + + load_template: function(name, ctx) { + return Mustache.render(this.elmt(name).html(), ctx); + }, + +}); diff --git a/manifold/static/js/record_generator.js b/manifold/static/js/record_generator.js new file mode 100644 index 00000000..f53c7ee9 --- /dev/null +++ b/manifold/static/js/record_generator.js @@ -0,0 +1,71 @@ +/* Buffered DOM updates */ +var RecordGenerator = Class.extend({ + + init: function(query, generators, number) + { + this._query = query; + this._generators = generators; + this._number = number; + }, + + random_int: function(options) + { + var default_options = { + max: 1000 + } + + if (typeof options == 'object') + options = $.extend(default_options, options); + else + options = default_options; + + return Math.floor(Math.random()*(options.max+1)); + }, + + random_string: function() + { + var default_options = { + possible: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + len: this.random_int({max: 15}) + } + + if (typeof options == 'object') + options = $.extend(default_options, options); + else + options = default_options; + + var text = ""; + + for( var i=0; i < options.len; i++ ) + text += options.possible.charAt(Math.floor(Math.random() * options.possible.length)); + + return text; + + }, + + generate_record: function() + { + var self = this; + var record = {}; + + $.each(this._query.fields, function(i, field) { + record[field] = self[self._generators[field]](); + }); + + // Publish records + manifold.raise_record_event(self._query.query_uuid, NEW_RECORD, record); + + }, + + run: function() + { + var record; + manifold.raise_record_event(this._query.query_uuid, CLEAR_RECORDS); + for (var i = 0; i < this._number; i++) { + record = this.generate_record(); + /* XXX publish record */ + } + manifold.raise_record_event(this._query.query_uuid, DONE); + + } +}); diff --git a/manifold/util/__init__.py b/manifold/util/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/manifold/util/autolog.py b/manifold/util/autolog.py new file mode 100644 index 00000000..e88b6f33 --- /dev/null +++ b/manifold/util/autolog.py @@ -0,0 +1,422 @@ +# Written by Brendan O'Connor, brenocon@gmail.com, www.anyall.org +# * Originally written Aug. 2005 +# * Posted to gist.github.com/16173 on Oct. 2008 + +# Copyright (c) 2003-2006 Open Source Applications Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re, sys, types + +""" +Have all your function & method calls automatically logged, in indented outline +form - unlike the stack snapshots in an interactive debugger, it tracks call +structure & stack depths across time! + +It hooks into all function calls that you specify, and logs each time they're +called. I find it especially useful when I don't know what's getting called +when, or need to continuously test for state changes. (by hacking this file) + +Originally inspired from the python cookbook: +http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/198078 + +Currently you can + - tag functions or individual methods to be autologged + - tag an entire class's methods to be autologged + - tag an entire module's classes and functions to be autologged + +TODO: + - allow tagging of ALL modules in the program on startup? + +CAVEATS: + - certain classes barf when you logclass() them -- most notably, + SWIG-generated wrappers, and perhaps others. + +USAGE: see examples on the bottom of this file. + + +Viewing tips +============ + +If your terminal can't keep up, try xterm or putty, they seem to be highest +performance. xterm is available for all platforms through X11... + +Also try: (RunChandler > log &); tail -f log + +Also, you can "less -R log" afterward and get the colors correct. + +If you have long lines, less -RS kills wrapping, enhancing readability. Also +can chop at formatAllArgs(). + +If you want long lines to be chopped realtime, try piping through less:: + + RunChandler | less -RS + +but then you have to hit 'space' lots to prevent chandler from freezing. +less's 'F' command is supposed to do this correctly but doesn't work for me. +""" + + +#@@@ should use the standard python logging system? +log = sys.stdout + +# Globally incremented across function calls, so tracks stack depth +indent = 0 +indStr = ' ' + + +# ANSI escape codes for terminals. +# X11 xterm: always works, all platforms +# cygwin dosbox: run through |cat and then colors work +# linux: works on console & gnome-terminal +# mac: untested + + +BLACK = "\033[0;30m" +BLUE = "\033[0;34m" +GREEN = "\033[0;32m" +CYAN = "\033[0;36m" +RED = "\033[0;31m" +PURPLE = "\033[0;35m" +BROWN = "\033[0;33m" +GRAY = "\033[0;37m" +BOLDGRAY = "\033[1;30m" +BOLDBLUE = "\033[1;34m" +BOLDGREEN = "\033[1;32m" +BOLDCYAN = "\033[1;36m" +BOLDRED = "\033[1;31m" +BOLDPURPLE = "\033[1;35m" +BOLDYELLOW = "\033[1;33m" +WHITE = "\033[1;37m" + +NORMAL = "\033[0m" + + +def indentlog(message): + global log, indStr, indent + print >>log, "%s%s" %(indStr*indent, message) + log.flush() + +def shortstr(obj): + """ + Where to put gritty heuristics to make an object appear in most useful + form. defaults to __str__. + """ + if "wx." in str(obj.__class__) or obj.__class__.__name__.startswith("wx"): + shortclassname = obj.__class__.__name__ + ##shortclassname = str(obj.__class__).split('.')[-1] + if hasattr(obj, "blockItem") and hasattr(obj.blockItem, "blockName"): + moreInfo = "block:'%s'" %obj.blockItem.blockName + else: + moreInfo = "at %d" %id(obj) + return "<%s %s>" % (shortclassname, moreInfo) + else: + return str(obj) + +def formatAllArgs(args, kwds): + """ + makes a nice string representation of all the arguments + """ + allargs = [] + for item in args: + allargs.append('%s' % shortstr(item)) + for key,item in kwds.items(): + allargs.append('%s=%s' % (key,shortstr(item))) + formattedArgs = ', '.join(allargs) + if len(formattedArgs) > 150: + return formattedArgs[:146] + " ..." + return formattedArgs + + +def logmodules(listOfModules): + for m in listOfModules: + bindmodule(m) + +def logmodule(module, logMatch=".*", logNotMatch="nomatchasfdasdf"): + """ + WARNING: this seems to break if you import SWIG wrapper classes + directly into the module namespace ... logclass() creates weirdness when + used on them, for some reason. + + @param module: could be either an actual module object, or the string + you can import (which seems to be the same thing as its + __name__). So you can say logmodule(__name__) at the end + of a module definition, to log all of it. + """ + + allow = lambda s: re.match(logMatch, s) and not re.match(logNotMatch, s) + + if isinstance(module, str): + d = {} + exec "import %s" % module in d + import sys + module = sys.modules[module] + + names = module.__dict__.keys() + for name in names: + if not allow(name): continue + + value = getattr(module, name) + if isinstance(value, type): + setattr(module, name, logclass(value)) + print>>log,"autolog.logmodule(): bound %s" %name + elif isinstance(value, types.FunctionType): + setattr(module, name, logfunction(value)) + print>>log,"autolog.logmodule(): bound %s" %name + +def logfunction(theFunction, displayName=None): + """a decorator.""" + if not displayName: displayName = theFunction.__name__ + + def _wrapper(*args, **kwds): + global indent + argstr = formatAllArgs(args, kwds) + + # Log the entry into the function + indentlog("%s%s%s (%s) " % (BOLDRED,displayName,NORMAL, argstr)) + log.flush() + + indent += 1 + returnval = theFunction(*args,**kwds) + indent -= 1 + + # Log return + ##indentlog("return: %s"% shortstr(returnval) + return returnval + return _wrapper + +def logmethod(theMethod, displayName=None): + """use this for class or instance methods, it formats with the object out front.""" + if not displayName: displayName = theMethod.__name__ + def _methodWrapper(self, *args, **kwds): + "Use this one for instance or class methods" + global indent + + argstr = formatAllArgs(args, kwds) + selfstr = shortstr(self) + + #print >> log,"%s%s. %s (%s) " % (indStr*indent,selfstr,methodname,argstr) + indentlog("%s.%s%s%s (%s) " % (selfstr, BOLDRED,theMethod.__name__,NORMAL, argstr)) + log.flush() + + indent += 1 + + if theMethod.__name__ == 'OnSize': + indentlog("position, size = %s%s %s%s" %(BOLDBLUE, self.GetPosition(), self.GetSize(), NORMAL)) + + returnval = theMethod(self, *args,**kwds) + + indent -= 1 + + return returnval + return _methodWrapper + + +def logclass(cls, methodsAsFunctions=False, + logMatch=".*", logNotMatch="asdfnomatch"): + """ + A class "decorator". But python doesn't support decorator syntax for + classes, so do it manually:: + + class C(object): + ... + C = logclass(C) + + @param methodsAsFunctions: set to True if you always want methodname first + in the display. Probably breaks if you're using class/staticmethods? + """ + + allow = lambda s: re.match(logMatch, s) and not re.match(logNotMatch, s) and \ + s not in ('__str__','__repr__') + + namesToCheck = cls.__dict__.keys() + + for name in namesToCheck: + if not allow(name): continue + # unbound methods show up as mere functions in the values of + # cls.__dict__,so we have to go through getattr + value = getattr(cls, name) + + if methodsAsFunctions and callable(value): + setattr(cls, name, logfunction(value)) + elif isinstance(value, types.MethodType): + #a normal instance method + if value.im_self == None: + setattr(cls, name, logmethod(value)) + + #class & static method are more complex. + #a class method + elif value.im_self == cls: + w = logmethod(value.im_func, + displayName="%s.%s" %(cls.__name__, value.__name__)) + setattr(cls, name, classmethod(w)) + else: assert False + + #a static method + elif isinstance(value, types.FunctionType): + w = logfunction(value, + displayName="%s.%s" %(cls.__name__, value.__name__)) + setattr(cls, name, staticmethod(w)) + return cls + +class LogMetaClass(type): + """ + Alternative to logclass(), you set this as a class's __metaclass__. + + It will not work if the metaclass has already been overridden (e.g. + schema.Item or zope.interface (used in Twisted) + + Also, it should fail for class/staticmethods, that hasnt been added here + yet. + """ + + def __new__(cls,classname,bases,classdict): + logmatch = re.compile(classdict.get('logMatch','.*')) + lognotmatch = re.compile(classdict.get('logNotMatch', 'nevermatchthisstringasdfasdf')) + + for attr,item in classdict.items(): + if callable(item) and logmatch.match(attr) and not lognotmatch.match(attr): + classdict['_H_%s'%attr] = item # rebind the method + classdict[attr] = logmethod(item) # replace method by wrapper + + return type.__new__(cls,classname,bases,classdict) + + + +# ---------------------------- Tests and examples ---------------------------- + +if __name__=='__main__': + print; print "------------------- single function logging ---------------" + @logfunction + def test(): + return 42 + + test() + + print; print "------------------- single method logging -----------------" + class Test1(object): + def __init__(self): + self.a = 10 + + @logmethod + def add(self,a,b): return a+b + + @logmethod + def fac(self,val): + if val == 1: + return 1 + else: + return val * self.fac(val-1) + + @logfunction + def fac2(self, val): + if val == 1: + return 1 + else: + return val * self.fac2(val-1) + + t = Test1() + t.add(5,6) + t.fac(4) + print "--- tagged as @logfunction, doesn't understand 'self' is special:" + t.fac2(4) + + + print; print """-------------------- class "decorator" usage ------------------""" + class Test2(object): + #will be ignored + def __init__(self): + self.a = 10 + def ignoreThis(self): pass + + + def add(self,a,b):return a+b + def fac(self,val): + if val == 1: + return 1 + else: + return val * self.fac(val-1) + + Test2 = logclass(Test2, logMatch='fac|add') + + t2 = Test2() + t2.add(5,6) + t2.fac(4) + t2.ignoreThis() + + + print; print "-------------------- metaclass usage ------------------" + class Test3(object): + __metaclass__ = LogMetaClass + logNotMatch = 'ignoreThis' + + def __init__(self): pass + + def fac(self,val): + if val == 1: + return 1 + else: + return val * self.fac(val-1) + def ignoreThis(self): pass + t3 = Test3() + t3.fac(4) + t3.ignoreThis() + + print; print "-------------- testing static & classmethods --------------" + class Test4(object): + @classmethod + def cm(cls, a, b): + print cls + return a+b + + def im(self, a, b): + print self + return a+b + + @staticmethod + def sm(a,b): return a+b + + Test4 = logclass(Test4) + + Test4.cm(4,3) + Test4.sm(4,3) + + t4 = Test4() + t4.im(4,3) + t4.sm(4,3) + t4.cm(4,3) + + #print; print "-------------- static & classmethods: where to put decorators? --------------" + #class Test5(object): + #@classmethod + #@logmethod + #def cm(cls, a, b): + #print cls + #return a+b + #@logmethod + #def im(self, a, b): + #print self + #return a+b + + #@staticmethod + #@logfunction + #def sm(a,b): return a+b + + + #Test5.cm(4,3) + #Test5.sm(4,3) + + #t5 = Test5() + #t5.im(4,3) + #t5.sm(4,3) + #t5.cm(4,3) diff --git a/manifold/util/callback.py b/manifold/util/callback.py new file mode 100644 index 00000000..03e82806 --- /dev/null +++ b/manifold/util/callback.py @@ -0,0 +1,49 @@ +from manifold.operators import LAST_RECORD +import threading + +#------------------------------------------------------------------ +# Class callback +#------------------------------------------------------------------ + +class Callback: + def __init__(self, deferred=None, router=None, cache_id=None): + #def __init__(self, deferred=None, event=None, router=None, cache_id=None): + self.results = [] + self._deferred = deferred + + #if not self.event: + self.event = threading.Event() + #else: + # self.event = event + + # Used for caching... + self.router = router + self.cache_id = cache_id + + def __call__(self, value): + # End of the list of records sent by Gateway + if value == LAST_RECORD: + if self.cache_id: + # Add query results to cache (expires in 30min) + #print "Result added to cached under id", self.cache_id + self.router.cache[self.cache_id] = (self.results, time.time() + CACHE_LIFETIME) + + if self._deferred: + # Send results back using deferred object + self._deferred.callback(self.results) + else: + # Not using deferred, trigger the event to return results + self.event.set() + return self.event + + # Not LAST_RECORD add the value to the results + self.results.append(value) + + def wait(self): + self.event.wait() + self.event.clear() + + def get_results(self): + self.wait() + return self.results + diff --git a/manifold/util/clause.py b/manifold/util/clause.py new file mode 100644 index 00000000..670a689a --- /dev/null +++ b/manifold/util/clause.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Implements a clause +# - a "tree" (more precisely a predecessor map, typically computed thanks to a DFS) +# - a set of needed fields (those queried by the user) +# +# Copyright (C) UPMC Paris Universitas +# Authors: +# Jordan Augé +# Marc-Olivier Buob + +import pyparsing as pp +import operator, re + +from manifold.util.predicate import Predicate +from types import StringTypes + +# XXX When to use Keyword vs. Regex vs. CaselessLiteral +# XXX capitalization ? + +# Instead of CaselessLiteral, try using CaselessKeyword. Keywords are better +# choice for grammar keywords, since they inherently avoid mistaking the leading +# 'in' of 'inside' as the keyword 'in' in your grammar. + + +class Clause(object): + + def __new__(cls, *args, **kwargs): + if len(args) == 1 and isinstance(args[0], StringTypes): + return ClauseStringParser().parse(args[0]) + return super(Clause, cls).__new__(cls, *args, **kwargs) + + def __init__(self, *args, **kwargs): + if len(args) == 2: + # unary + self.operator = Predicate.operators[args[0]] + self.operands = [args[1]] + elif len(args) == 3: + self.operator = Predicate.operators[args[1]] + self.operands = [args[0], args[2]] + else: + raise Exception, "Clause can only be unary or binary" + + def opstr(self, operator): + ops = [string for string, op in Predicate.operators.items() if op == operator] + return ops[0] if ops else '' + + def __repr__(self): + if len(self.operands) == 1: + return "%s(%s)" % (self.operator, self.operands[0]) + else: + return "(%s %s %s)" % (self.operands[0], self.opstr(self.operator), self.operands[1]) + +class ClauseStringParser(object): + + def __init__(self): + """ + BNF HERE + """ + + #integer = pp.Word(nums) + #floatNumber = pp.Regex(r'\d+(\.\d*)?([eE]\d+)?') + point = pp.Literal( "." ) + e = pp.CaselessLiteral( "E" ) + + # Regex string representing the set of possible operators + # Example : ">=|<=|!=|>|<|=" + OPERATOR_RX = '|'.join([re.sub('\|', '\|', o) for o in Predicate.operators.keys()]) + + # predicate + field = pp.Word(pp.alphanums + '_') + operator = pp.Regex(OPERATOR_RX).setName("operator") + value = pp.QuotedString('"') #| pp.Combine( pp.Word( "+-"+ pp.nums, pp.nums) + pp.Optional( point + pp.Optional( pp.Word( pp.nums ) ) ) + pp.Optional( e + pp.Word( "+-"+pp.nums, pp.nums ) ) ) + + predicate = (field + operator + value).setParseAction(self.handlePredicate) + + # clause of predicates + and_op = pp.CaselessLiteral("and") | pp.Keyword("&&") + or_op = pp.CaselessLiteral("or") | pp.Keyword("||") + not_op = pp.Keyword("!") + + predicate_precedence_list = [ + (not_op, 1, pp.opAssoc.RIGHT, lambda x: self.handleClause(*x)), + (and_op, 2, pp.opAssoc.LEFT, lambda x: self.handleClause(*x)), + (or_op, 2, pp.opAssoc.LEFT, lambda x: self.handleClause(*x)) + ] + clause = pp.operatorPrecedence(predicate, predicate_precedence_list) + + self.bnf = clause + + def handlePredicate(self, args): + return Predicate(*args) + + def handleClause(self, args): + return Clause(*args) + + def parse(self, string): + return self.bnf.parseString(string,parseAll=True) + +if __name__ == "__main__": + print ClauseStringParser().parse('country == "Europe" || ts > "01-01-2007" && country == "France"') + print Clause('country == "Europe" || ts > "01-01-2007" && country == "France"') diff --git a/manifold/util/colors.py b/manifold/util/colors.py new file mode 100644 index 00000000..82639bbf --- /dev/null +++ b/manifold/util/colors.py @@ -0,0 +1,38 @@ +# ANSI escape codes for terminals. +# X11 xterm: always works, all platforms +# cygwin dosbox: run through |cat and then colors work +# linux: works on console & gnome-terminal +# mac: untested + +BLACK = "\033[0;30m" +BLUE = "\033[0;34m" +GREEN = "\033[0;32m" +CYAN = "\033[0;36m" +RED = "\033[0;31m" +PURPLE = "\033[0;35m" +BROWN = "\033[0;33m" +GRAY = "\033[0;37m" +BOLDGRAY = "\033[1;30m" +BOLDBLUE = "\033[1;34m" +BOLDGREEN = "\033[1;32m" +BOLDCYAN = "\033[1;36m" +BOLDRED = "\033[1;31m" +BOLDPURPLE = "\033[1;35m" +BOLDYELLOW = "\033[1;33m" +WHITE = "\033[1;37m" + +MYGREEN = '\033[92m' +MYBLUE = '\033[94m' +MYWARNING = '\033[93m' +MYRED = '\033[91m' +MYHEADER = '\033[95m' +MYEND = '\033[0m' + +NORMAL = "\033[0m" + +if __name__ == '__main__': + # Display color names in their color + for name, color in locals().items(): + if name.startswith('__'): continue + print color, name, MYEND + diff --git a/manifold/util/daemon.py b/manifold/util/daemon.py new file mode 100644 index 00000000..2e5d760e --- /dev/null +++ b/manifold/util/daemon.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Daemon: superclass used to implement a daemon easily +# +# Copyright (C)2009-2012, UPMC Paris Universitas +# Authors: +# Marc-Olivier Buob + +# see also: http://www.jejik.com/files/examples/daemon3x.py + +# This is used to import the daemon package instead of the local module which is +# named identically... +from __future__ import absolute_import + +from manifold.util.singleton import Singleton +from manifold.util.log import Log +from manifold.util.options import Options + +import atexit, os, signal, lockfile, logging, sys + +class Daemon(object): + __metaclass__ = Singleton + + DEFAULTS = { + # Running + "uid" : os.getuid(), + "gid" : os.getgid(), + "working_directory" : "/", + "debugmode" : False, + "no_daemon" : False, + "pid_filename" : "/var/run/%s.pid" % Options().get_name() + } + + #------------------------------------------------------------------------- + # Checks + #------------------------------------------------------------------------- + + def check_python_daemon(self): + """ + \brief Check whether python-daemon is properly installed + \return True if everything is file, False otherwise + """ + # http://www.python.org/dev/peps/pep-3143/ + ret = False + try: + import daemon + getattr(daemon, "DaemonContext") + ret = True + except AttributeError, e: + print e + # daemon and python-daemon conflict with each other + Log.critical("Please install python-daemon instead of daemon. Remove daemon first.") + except ImportError: + Log.critical("Please install python-daemon - easy_install python-daemon.") + return ret + + #------------------------------------------------------------------------ + # Initialization + #------------------------------------------------------------------------ + + def make_handler_rsyslog(self, rsyslog_host, rsyslog_port, log_level): + """ + \brief (Internal usage) Prepare logging via rsyslog + \param rsyslog_host The hostname of the rsyslog server + \param rsyslog_port The port of the rsyslog server + \param log_level Log level + """ + # Prepare the handler + shandler = handlers.SysLogHandler( + (rsyslog_host, rsyslog_port), + facility = handlers.SysLogHandler.LOG_DAEMON + ) + + # The log file must remain open while daemonizing + self.files_to_keep.append(shandler.socket) + self.prepare_handler(shandler, log_level) + return shandler + + def make_handler_locallog(self, log_filename, log_level): + """ + \brief (Internal usage) Prepare local logging + \param log_filename The file in which we write the logs + \param log_level Log level + """ + # Create directory in which we store the log file + log_dir = os.path.dirname(log_filename) + if not os.path.exists(log_dir): + try: + os.makedirs(log_dir) + except OSError, why: + log_error("OS error: %s" % why) + + # Prepare the handler + shandler = logging.handlers.RotatingFileHandler( + log_filename, + backupCount = 0 + ) + + # The log file must remain open while daemonizing + self.files_to_keep.append(shandler.stream) + self.prepare_handler(shandler, log_level) + return shandler + + def prepare_handler(self, shandler, log_level): + """ + \brief (Internal usage) + \param shandler Handler used to log information + \param log_level Log level + """ + shandler.setLevel(log_level) + formatter = logging.Formatter("%(asctime)s: %(name)s: %(levelname)s %(message)s") + shandler.setFormatter(formatter) + self.log.addHandler(shandler) + self.log.setLevel(getattr(logging, log_level, logging.INFO)) + + def __init__( + self, + #daemon_name, + terminate_callback = None + #uid = os.getuid(), + #gid = os.getgid(), + #working_directory = "/", + #pid_filename = None, + #no_daemon = False, + #debug = False, + #log = None, # logging.getLogger("plop") + #rsyslog_host = "localhost", # Pass None if no rsyslog server + #rsyslog_port = 514, + #log_file = None, + #log_level = logging.INFO + ): + """ + \brief Constructor + \param daemon_name The name of the daemon + \param uid UID used to run the daemon + \param gid GID used to run the daemon + \param working_directory Working directory used to run the daemon. + Example: /var/lib/foo/ + \param pid_filename Absolute path of the PID file + Example: /var/run/foo.pid + (ignored if no_daemon == True) + \param no_daemon Do not detach the daemon from the terminal + \param debug Run daemon in debug mode + \param log The logger, pass None if unused + Example: logging.getLogger('foo')) + \param rsyslog_host Rsyslog hostname, pass None if unused. + If rsyslog_host is set to None, log are stored locally + \param rsyslog_port Rsyslog port + \param log_file Absolute path of the local log file. + Example: /var/log/foo.log) + \param log_level Log level + Example: logging.INFO + """ + + # Daemon parameters + #self.daemon_name = daemon_name + self.terminate_callback = terminate_callback + #Options().uid = uid + #Options().gid = gid + #Options().working_directory = working_directory + #self.pid_filename = None if no_daemon else pid_filename + #Options().no_daemon = no_daemon + #Options().lock_file = None + #Options().debug = debug + #self.log = log + #self.rsyslog_host = rsyslog_host + #self.rsyslog_port = rsyslog_port + #self.log_file = log_file + #self.log_level = log_level + + # Reference which file descriptors must remain opened while + # daemonizing (for instance the file descriptor related to + # the logger) + self.files_to_keep = [] + + # Initialize self.log (require self.files_to_keep) + #if self.log: # for debugging by using stdout, log may be equal to None + # if rsyslog_host: + # shandler = self.make_handler_rsyslog( + # rsyslog_host, + # rsyslog_port, + # log_level + # ) + # elif log_file: + # shandler = self.make_handler_locallog( + # log_file, + # log_level + # ) + + @classmethod + def init_options(self): + opt = Options() + + opt.add_option( + "--uid", dest = "uid", + help = "UID used to run the dispatcher.", + default = self.DEFAULTS['uid'] + ) + opt.add_option( + "--gid", dest = "gid", + help = "GID used to run the dispatcher.", + default = self.DEFAULTS['gid'] + ) + opt.add_option( + "-w", "--working-directory", dest = "working_directory", + help = "Working directory.", + default = self.DEFAULTS['working_directory'] + ) + opt.add_option( + "-D", "--debugmode", action = "store_false", dest = "debugmode", + help = "Daemon debug mode (useful for developers).", + default = self.DEFAULTS['debugmode'] + ) + opt.add_option( + "-n", "--no-daemon", action = "store_true", dest = "no_daemon", + help = "Run as daemon (detach from terminal).", + default = self.DEFAULTS["no_daemon"] + ) + opt.add_option( + "-i", "--pid-file", dest = "pid_filename", + help = "Absolute path to the pid-file to use when running as daemon.", + default = self.DEFAULTS['pid_filename'] + ) + + + + #------------------------------------------------------------------------ + # Daemon stuff + #------------------------------------------------------------------------ + + def remove_pid_file(self): + """ + \brief Remove the pid file (internal usage) + """ + # The lock file is implicitely released while removing the pid file + Log.debug("Removing %s" % Options().pid_filename) + if os.path.exists(Options().pid_filename) == True: + os.remove(Options().pid_filename) + + def make_pid_file(self): + """ + \brief Create a pid file in which we store the PID of the daemon if needed + """ + if Options().pid_filename and Options().no_daemon == False: + atexit.register(self.remove_pid_file) + file(Options().pid_filename, "w+").write("%s\n" % str(os.getpid())) + + def get_pid_from_pid_file(self): + """ + \brief Retrieve the PID of the daemon thanks to the pid file. + \return None if the pid file is not readable or does not exists + """ + pid = None + if Options().pid_filename: + try: + f_pid = file(Options().pid_filename, "r") + pid = int(f_pid.read().strip()) + f_pid.close() + except IOError: + pid = None + return pid + + def make_lock_file(self): + """ + \brief Prepare the lock file required to manage the pid file + Initialize Options().lock_file + """ + if Options().pid_filename and Options().no_daemon == False: + Log.debug("Daemonizing using pid file '%s'" % Options().pid_filename) + Options().lock_file = lockfile.FileLock(Options().pid_filename) + if Options().lock_file.is_locked() == True: + log_error("'%s' is already running ('%s' is locked)." % (Options().get_name(), Options().pid_filename)) + self.terminate() + Options().lock_file.acquire() + else: + Options().lock_file = None + + def start(self): + """ + \brief Start the daemon + """ + # Check whether daemon module is properly installed + if self.check_python_daemon() == False: + self.terminate() + import daemon + + # Prepare Options().lock_file + self.make_lock_file() + + # Prepare the daemon context + dcontext = daemon.DaemonContext( + detach_process = (not Options().no_daemon), + working_directory = Options().working_directory, + pidfile = Options().lock_file if not Options().no_daemon else None, + stdin = sys.stdin, + stdout = sys.stdout, + stderr = sys.stderr, + uid = Options().uid, + gid = Options().gid, + files_preserve = Log().files_to_keep + ) + + # Prepare signal handling to stop properly if the daemon is killed + # Note that signal.SIGKILL can't be handled: + # http://crunchtools.com/unixlinux-signals-101/ + dcontext.signal_map = { + signal.SIGTERM : self.signal_handler, + signal.SIGQUIT : self.signal_handler, + signal.SIGINT : self.signal_handler + } + + if Options().debugmode == True: + self.main() + else: + with dcontext: + self.make_pid_file() + try: + self.main() + except Exception, why: + Log.error("Unhandled exception in start: %s" % why) + + def signal_handler(self, signal_id, frame): + """ + \brief Stop the daemon (signal handler) + The lockfile is implicitly released by the daemon package + \param signal_id The integer identifying the signal + (see also "man 7 signal") + Example: 15 if the received signal is signal.SIGTERM + \param frame + """ + self.terminate() + + def stop(self): + Log.debug("Stopping '%s'" % self.daemon_name) + + def terminate(self): + if self.terminate_callback: + self.terminate_callback() + else: + sys.exit(0) + +Daemon.init_options() diff --git a/manifold/util/dfs.py b/manifold/util/dfs.py new file mode 100644 index 00000000..019645d6 --- /dev/null +++ b/manifold/util/dfs.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Depth first search algorithm +# Based on http://www.boost.org/doc/libs/1_52_0/libs/graph/doc/depth_first_search.html +# +# Copyright (C) UPMC Paris Universitas +# Authors: +# Marc-Olivier Buob +# Jordan Augé + +class dfs_color: + WHITE = 1 # not yet visited + GRAY = 2 # currently visited + BLACK = 3 # visited + +#DFS(G) +# for each vertex u in V +# color[u] := WHITE +# p[u] = u +# end for +# time := 0 +# if there is a starting vertex s +# call DFS-VISIT(G, s) +# for each vertex u in V +# if color[u] = WHITE +# call DFS-VISIT(G, u) +# end for +# return (p,d_time,f_time) + +def dfs(graph, root, exclude_uv=None): + """ + \brief Run the DFS algorithm + \param graph The graph we explore + \param root The starting vertex + \return A dictionnary which maps each vertex of the tree + to its predecessor, None otherwise. + Only the root node as a predecessor equal to None. + Nodes not referenced in this dictionnary do not + belong to the tree. + """ + # Initialization + map_vertex_color = {} + map_vertex_pred = {} + for u in graph.nodes(): + map_vertex_color[u] = dfs_color.WHITE + map_vertex_pred[u] = None + + # Recursive calls + if not exclude_uv: + exclude_uv = lambda u,v: False + dfs_visit(graph, root, map_vertex_color, map_vertex_pred, exclude_uv) + + # Remove from map_vertex_pred the vertices having no + # predecessor but the root node. + for v, u in map_vertex_pred.items(): + if u == None and v != root: + del map_vertex_pred[v] + + return map_vertex_pred + +#DFS-VISIT(G, u) +# color[u] := GRAY +# d_time[u] := time := time + 1 +# for each v in Adj[u] +# if (color[v] = WHITE) +# p[v] = u +# call DFS-VISIT(G, v) +# else if (color[v] = GRAY) +# ... +# else if (color[v] = BLACK) +# ... +# end for +# color[u] := BLACK +# f_time[u] := time := time + 1 + +def dfs_visit(graph, u, map_vertex_color, map_vertex_pred, exclude_uv): + """ + \brief Internal usage (DFS implementation) + \param graph The graph we explore + \param u The current node + \param map_vertex_color: maps each vertex to a color + - dfs_color.WHITE: iif the vertex is not reachable from the root node + - dfs_color.BLACK: otherwise + \param map_vertex_pred: maps each vertex to its predecessor (if any) visited + during the DFS exploration, None otherwise + """ + map_vertex_color[u] = dfs_color.GRAY + for v in graph.successors(u): + color_v = map_vertex_color[v] + if color_v == dfs_color.WHITE and not exclude_uv(u, v): + map_vertex_pred[v] = u + dfs_visit(graph, v, map_vertex_color, map_vertex_pred, exclude_uv) + map_vertex_color[u] = dfs_color.BLACK + diff --git a/manifold/util/enum.py b/manifold/util/enum.py new file mode 100644 index 00000000..4f3c577b --- /dev/null +++ b/manifold/util/enum.py @@ -0,0 +1,7 @@ +class Enum(object): + def __init__(self, *keys): + self.__dict__.update(zip(keys, range(len(keys)))) + self.invmap = {v:k for k, v in self.__dict__.items()} + + def get_str(self, value): + return self.invmap[value] diff --git a/manifold/util/frozendict.py b/manifold/util/frozendict.py new file mode 100644 index 00000000..32902cb7 --- /dev/null +++ b/manifold/util/frozendict.py @@ -0,0 +1,47 @@ +import copy + +class frozendict(dict): + def _blocked_attribute(obj): + raise AttributeError, "A frozendict cannot be modified." + _blocked_attribute = property(_blocked_attribute) + + __delitem__ = __setitem__ = clear = _blocked_attribute + pop = popitem = setdefault = update = _blocked_attribute + + def __new__(cls, *args, **kw): + new = dict.__new__(cls) + + args_ = [] + for arg in args: + if isinstance(arg, dict): + arg = copy.copy(arg) + for k, v in arg.items(): + if isinstance(v, dict): + arg[k] = frozendict(v) + elif isinstance(v, list): + v_ = list() + for elm in v: + if isinstance(elm, dict): + v_.append( frozendict(elm) ) + else: + v_.append( elm ) + arg[k] = tuple(v_) + args_.append( arg ) + else: + args_.append( arg ) + + dict.__init__(new, *args_, **kw) + return new + + def __init__(self, *args, **kw): + pass + + def __hash__(self): + try: + return self._cached_hash + except AttributeError: + h = self._cached_hash = hash(tuple(sorted(self.items()))) + return h + + def __repr__(self): + return "frozendict(%s)" % dict.__repr__(self) diff --git a/manifold/util/functional.py b/manifold/util/functional.py new file mode 100644 index 00000000..a4119398 --- /dev/null +++ b/manifold/util/functional.py @@ -0,0 +1,53 @@ +"""Borrowed from Django.""" + +from threading import Lock + +class LazyObject(object): + """ + A wrapper for another class that can be used to delay instantiation of the + wrapped class. + + By subclassing, you have the opportunity to intercept and alter the + instantiation. If you don't need to do that, use SimpleLazyObject. + """ + def __init__(self): + self._wrapped = None + self._lock = Lock() + + def __getattr__(self, name): + self._lock.acquire() + if self._wrapped is None: + self._setup() + self._lock.release() + return getattr(self._wrapped, name) + + def __setattr__(self, name, value): + if name in ["_wrapped", "_lock"]: + # Assign to __dict__ to avoid infinite __setattr__ loops. + self.__dict__[name] = value + else: + if self._wrapped is None: + self._setup() + setattr(self._wrapped, name, value) + + def __delattr__(self, name): + if name == "_wrapped": + raise TypeError("can't delete _wrapped.") + if self._wrapped is None: + self._setup() + delattr(self._wrapped, name) + + def _setup(self): + """ + Must be implemented by subclasses to initialise the wrapped object. + """ + raise NotImplementedError + + # introspection support: + __members__ = property(lambda self: self.__dir__()) + + def __dir__(self): + if self._wrapped is None: + self._setup() + return dir(self._wrapped) + diff --git a/manifold/util/ipaddr.py b/manifold/util/ipaddr.py new file mode 100644 index 00000000..ad27ae9d --- /dev/null +++ b/manifold/util/ipaddr.py @@ -0,0 +1,1897 @@ +#!/usr/bin/python +# +# Copyright 2007 Google Inc. +# Licensed to PSF under a Contributor Agreement. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""A fast, lightweight IPv4/IPv6 manipulation library in Python. + +This library is used to create/poke/manipulate IPv4 and IPv6 addresses +and networks. + +""" + +__version__ = '2.1.10' + +import struct + +IPV4LENGTH = 32 +IPV6LENGTH = 128 + + +class AddressValueError(ValueError): + """A Value Error related to the address.""" + + +class NetmaskValueError(ValueError): + """A Value Error related to the netmask.""" + + +def IPAddress(address, version=None): + """Take an IP string/int and return an object of the correct type. + + Args: + address: A string or integer, the IP address. Either IPv4 or + IPv6 addresses may be supplied; integers less than 2**32 will + be considered to be IPv4 by default. + version: An Integer, 4 or 6. If set, don't try to automatically + determine what the IP address type is. important for things + like IPAddress(1), which could be IPv4, '0.0.0.1', or IPv6, + '::1'. + + Returns: + An IPv4Address or IPv6Address object. + + Raises: + ValueError: if the string passed isn't either a v4 or a v6 + address. + + """ + if version: + if version == 4: + return IPv4Address(address) + elif version == 6: + return IPv6Address(address) + + try: + return IPv4Address(address) + except (AddressValueError, NetmaskValueError): + pass + + try: + return IPv6Address(address) + except (AddressValueError, NetmaskValueError): + pass + + raise ValueError('%r does not appear to be an IPv4 or IPv6 address' % + address) + + +def IPNetwork(address, version=None, strict=False): + """Take an IP string/int and return an object of the correct type. + + Args: + address: A string or integer, the IP address. Either IPv4 or + IPv6 addresses may be supplied; integers less than 2**32 will + be considered to be IPv4 by default. + version: An Integer, if set, don't try to automatically + determine what the IP address type is. important for things + like IPNetwork(1), which could be IPv4, '0.0.0.1/32', or IPv6, + '::1/128'. + + Returns: + An IPv4Network or IPv6Network object. + + Raises: + ValueError: if the string passed isn't either a v4 or a v6 + address. Or if a strict network was requested and a strict + network wasn't given. + + """ + if version: + if version == 4: + return IPv4Network(address, strict) + elif version == 6: + return IPv6Network(address, strict) + + try: + return IPv4Network(address, strict) + except (AddressValueError, NetmaskValueError): + pass + + try: + return IPv6Network(address, strict) + except (AddressValueError, NetmaskValueError): + pass + + raise ValueError('%r does not appear to be an IPv4 or IPv6 network' % + address) + + +def v4_int_to_packed(address): + """The binary representation of this address. + + Args: + address: An integer representation of an IPv4 IP address. + + Returns: + The binary representation of this address. + + Raises: + ValueError: If the integer is too large to be an IPv4 IP + address. + """ + if address > _BaseV4._ALL_ONES: + raise ValueError('Address too large for IPv4') + return Bytes(struct.pack('!I', address)) + + +def v6_int_to_packed(address): + """The binary representation of this address. + + Args: + address: An integer representation of an IPv4 IP address. + + Returns: + The binary representation of this address. + """ + return Bytes(struct.pack('!QQ', address >> 64, address & (2**64 - 1))) + + +def _find_address_range(addresses): + """Find a sequence of addresses. + + Args: + addresses: a list of IPv4 or IPv6 addresses. + + Returns: + A tuple containing the first and last IP addresses in the sequence. + + """ + first = last = addresses[0] + for ip in addresses[1:]: + if ip._ip == last._ip + 1: + last = ip + else: + break + return (first, last) + +def _get_prefix_length(number1, number2, bits): + """Get the number of leading bits that are same for two numbers. + + Args: + number1: an integer. + number2: another integer. + bits: the maximum number of bits to compare. + + Returns: + The number of leading bits that are the same for two numbers. + + """ + for i in range(bits): + if number1 >> i == number2 >> i: + return bits - i + return 0 + +def _count_righthand_zero_bits(number, bits): + """Count the number of zero bits on the right hand side. + + Args: + number: an integer. + bits: maximum number of bits to count. + + Returns: + The number of zero bits on the right hand side of the number. + + """ + if number == 0: + return bits + for i in range(bits): + if (number >> i) % 2: + return i + +def summarize_address_range(first, last): + """Summarize a network range given the first and last IP addresses. + + Example: + >>> summarize_address_range(IPv4Address('1.1.1.0'), + IPv4Address('1.1.1.130')) + [IPv4Network('1.1.1.0/25'), IPv4Network('1.1.1.128/31'), + IPv4Network('1.1.1.130/32')] + + Args: + first: the first IPv4Address or IPv6Address in the range. + last: the last IPv4Address or IPv6Address in the range. + + Returns: + The address range collapsed to a list of IPv4Network's or + IPv6Network's. + + Raise: + TypeError: + If the first and last objects are not IP addresses. + If the first and last objects are not the same version. + ValueError: + If the last object is not greater than the first. + If the version is not 4 or 6. + + """ + if not (isinstance(first, _BaseIP) and isinstance(last, _BaseIP)): + raise TypeError('first and last must be IP addresses, not networks') + if first.version != last.version: + raise TypeError("%s and %s are not of the same version" % ( + str(first), str(last))) + if first > last: + raise ValueError('last IP address must be greater than first') + + networks = [] + + if first.version == 4: + ip = IPv4Network + elif first.version == 6: + ip = IPv6Network + else: + raise ValueError('unknown IP version') + + ip_bits = first._max_prefixlen + first_int = first._ip + last_int = last._ip + while first_int <= last_int: + nbits = _count_righthand_zero_bits(first_int, ip_bits) + current = None + while nbits >= 0: + addend = 2**nbits - 1 + current = first_int + addend + nbits -= 1 + if current <= last_int: + break + prefix = _get_prefix_length(first_int, current, ip_bits) + net = ip('%s/%d' % (str(first), prefix)) + networks.append(net) + if current == ip._ALL_ONES: + break + first_int = current + 1 + first = IPAddress(first_int, version=first._version) + return networks + +def _collapse_address_list_recursive(addresses): + """Loops through the addresses, collapsing concurrent netblocks. + + Example: + + ip1 = IPv4Network('1.1.0.0/24') + ip2 = IPv4Network('1.1.1.0/24') + ip3 = IPv4Network('1.1.2.0/24') + ip4 = IPv4Network('1.1.3.0/24') + ip5 = IPv4Network('1.1.4.0/24') + ip6 = IPv4Network('1.1.0.1/22') + + _collapse_address_list_recursive([ip1, ip2, ip3, ip4, ip5, ip6]) -> + [IPv4Network('1.1.0.0/22'), IPv4Network('1.1.4.0/24')] + + This shouldn't be called directly; it is called via + collapse_address_list([]). + + Args: + addresses: A list of IPv4Network's or IPv6Network's + + Returns: + A list of IPv4Network's or IPv6Network's depending on what we were + passed. + + """ + ret_array = [] + optimized = False + + for cur_addr in addresses: + if not ret_array: + ret_array.append(cur_addr) + continue + if cur_addr in ret_array[-1]: + optimized = True + elif cur_addr == ret_array[-1].supernet().subnet()[1]: + ret_array.append(ret_array.pop().supernet()) + optimized = True + else: + ret_array.append(cur_addr) + + if optimized: + return _collapse_address_list_recursive(ret_array) + + return ret_array + + +def collapse_address_list(addresses): + """Collapse a list of IP objects. + + Example: + collapse_address_list([IPv4('1.1.0.0/24'), IPv4('1.1.1.0/24')]) -> + [IPv4('1.1.0.0/23')] + + Args: + addresses: A list of IPv4Network or IPv6Network objects. + + Returns: + A list of IPv4Network or IPv6Network objects depending on what we + were passed. + + Raises: + TypeError: If passed a list of mixed version objects. + + """ + i = 0 + addrs = [] + ips = [] + nets = [] + + # split IP addresses and networks + for ip in addresses: + if isinstance(ip, _BaseIP): + if ips and ips[-1]._version != ip._version: + raise TypeError("%s and %s are not of the same version" % ( + str(ip), str(ips[-1]))) + ips.append(ip) + elif ip._prefixlen == ip._max_prefixlen: + if ips and ips[-1]._version != ip._version: + raise TypeError("%s and %s are not of the same version" % ( + str(ip), str(ips[-1]))) + ips.append(ip.ip) + else: + if nets and nets[-1]._version != ip._version: + raise TypeError("%s and %s are not of the same version" % ( + str(ip), str(ips[-1]))) + nets.append(ip) + + # sort and dedup + ips = sorted(set(ips)) + nets = sorted(set(nets)) + + while i < len(ips): + (first, last) = _find_address_range(ips[i:]) + i = ips.index(last) + 1 + addrs.extend(summarize_address_range(first, last)) + + return _collapse_address_list_recursive(sorted( + addrs + nets, key=_BaseNet._get_networks_key)) + +# backwards compatibility +CollapseAddrList = collapse_address_list + +# We need to distinguish between the string and packed-bytes representations +# of an IP address. For example, b'0::1' is the IPv4 address 48.58.58.49, +# while '0::1' is an IPv6 address. +# +# In Python 3, the native 'bytes' type already provides this functionality, +# so we use it directly. For earlier implementations where bytes is not a +# distinct type, we create a subclass of str to serve as a tag. +# +# Usage example (Python 2): +# ip = ipaddr.IPAddress(ipaddr.Bytes('xxxx')) +# +# Usage example (Python 3): +# ip = ipaddr.IPAddress(b'xxxx') +try: + if bytes is str: + raise TypeError("bytes is not a distinct type") + Bytes = bytes +except (NameError, TypeError): + class Bytes(str): + def __repr__(self): + return 'Bytes(%s)' % str.__repr__(self) + +def get_mixed_type_key(obj): + """Return a key suitable for sorting between networks and addresses. + + Address and Network objects are not sortable by default; they're + fundamentally different so the expression + + IPv4Address('1.1.1.1') <= IPv4Network('1.1.1.1/24') + + doesn't make any sense. There are some times however, where you may wish + to have ipaddr sort these for you anyway. If you need to do this, you + can use this function as the key= argument to sorted(). + + Args: + obj: either a Network or Address object. + Returns: + appropriate key. + + """ + if isinstance(obj, _BaseNet): + return obj._get_networks_key() + elif isinstance(obj, _BaseIP): + return obj._get_address_key() + return NotImplemented + +class _IPAddrBase(object): + + """The mother class.""" + + def __index__(self): + return self._ip + + def __int__(self): + return self._ip + + def __hex__(self): + return hex(self._ip) + + @property + def exploded(self): + """Return the longhand version of the IP address as a string.""" + return self._explode_shorthand_ip_string() + + @property + def compressed(self): + """Return the shorthand version of the IP address as a string.""" + return str(self) + + +class _BaseIP(_IPAddrBase): + + """A generic IP object. + + This IP class contains the version independent methods which are + used by single IP addresses. + + """ + + def __eq__(self, other): + try: + return (self._ip == other._ip + and self._version == other._version) + except AttributeError: + return NotImplemented + + def __ne__(self, other): + eq = self.__eq__(other) + if eq is NotImplemented: + return NotImplemented + return not eq + + def __le__(self, other): + gt = self.__gt__(other) + if gt is NotImplemented: + return NotImplemented + return not gt + + def __ge__(self, other): + lt = self.__lt__(other) + if lt is NotImplemented: + return NotImplemented + return not lt + + def __lt__(self, other): + if self._version != other._version: + raise TypeError('%s and %s are not of the same version' % ( + str(self), str(other))) + if not isinstance(other, _BaseIP): + raise TypeError('%s and %s are not of the same type' % ( + str(self), str(other))) + if self._ip != other._ip: + return self._ip < other._ip + return False + + def __gt__(self, other): + if self._version != other._version: + raise TypeError('%s and %s are not of the same version' % ( + str(self), str(other))) + if not isinstance(other, _BaseIP): + raise TypeError('%s and %s are not of the same type' % ( + str(self), str(other))) + if self._ip != other._ip: + return self._ip > other._ip + return False + + # Shorthand for Integer addition and subtraction. This is not + # meant to ever support addition/subtraction of addresses. + def __add__(self, other): + if not isinstance(other, int): + return NotImplemented + return IPAddress(int(self) + other, version=self._version) + + def __sub__(self, other): + if not isinstance(other, int): + return NotImplemented + return IPAddress(int(self) - other, version=self._version) + + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, str(self)) + + def __str__(self): + return '%s' % self._string_from_ip_int(self._ip) + + def __hash__(self): + return hash(hex(long(self._ip))) + + def _get_address_key(self): + return (self._version, self) + + @property + def version(self): + raise NotImplementedError('BaseIP has no version') + + +class _BaseNet(_IPAddrBase): + + """A generic IP object. + + This IP class contains the version independent methods which are + used by networks. + + """ + + def __init__(self, address): + self._cache = {} + + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, str(self)) + + def iterhosts(self): + """Generate Iterator over usable hosts in a network. + + This is like __iter__ except it doesn't return the network + or broadcast addresses. + + """ + cur = int(self.network) + 1 + bcast = int(self.broadcast) - 1 + while cur <= bcast: + cur += 1 + yield IPAddress(cur - 1, version=self._version) + + def __iter__(self): + cur = int(self.network) + bcast = int(self.broadcast) + while cur <= bcast: + cur += 1 + yield IPAddress(cur - 1, version=self._version) + + def __getitem__(self, n): + network = int(self.network) + broadcast = int(self.broadcast) + if n >= 0: + if network + n > broadcast: + raise IndexError + return IPAddress(network + n, version=self._version) + else: + n += 1 + if broadcast + n < network: + raise IndexError + return IPAddress(broadcast + n, version=self._version) + + def __lt__(self, other): + if self._version != other._version: + raise TypeError('%s and %s are not of the same version' % ( + str(self), str(other))) + if not isinstance(other, _BaseNet): + raise TypeError('%s and %s are not of the same type' % ( + str(self), str(other))) + if self.network != other.network: + return self.network < other.network + if self.netmask != other.netmask: + return self.netmask < other.netmask + return False + + def __gt__(self, other): + if self._version != other._version: + raise TypeError('%s and %s are not of the same version' % ( + str(self), str(other))) + if not isinstance(other, _BaseNet): + raise TypeError('%s and %s are not of the same type' % ( + str(self), str(other))) + if self.network != other.network: + return self.network > other.network + if self.netmask != other.netmask: + return self.netmask > other.netmask + return False + + def __le__(self, other): + gt = self.__gt__(other) + if gt is NotImplemented: + return NotImplemented + return not gt + + def __ge__(self, other): + lt = self.__lt__(other) + if lt is NotImplemented: + return NotImplemented + return not lt + + def __eq__(self, other): + try: + return (self._version == other._version + and self.network == other.network + and int(self.netmask) == int(other.netmask)) + except AttributeError: + if isinstance(other, _BaseIP): + return (self._version == other._version + and self._ip == other._ip) + + def __ne__(self, other): + eq = self.__eq__(other) + if eq is NotImplemented: + return NotImplemented + return not eq + + def __str__(self): + return '%s/%s' % (str(self.ip), + str(self._prefixlen)) + + def __hash__(self): + return hash(int(self.network) ^ int(self.netmask)) + + def __contains__(self, other): + # always false if one is v4 and the other is v6. + if self._version != other._version: + return False + # dealing with another network. + if isinstance(other, _BaseNet): + return (self.network <= other.network and + self.broadcast >= other.broadcast) + # dealing with another address + else: + return (int(self.network) <= int(other._ip) <= + int(self.broadcast)) + + def overlaps(self, other): + """Tell if self is partly contained in other.""" + return self.network in other or self.broadcast in other or ( + other.network in self or other.broadcast in self) + + @property + def network(self): + x = self._cache.get('network') + if x is None: + x = IPAddress(self._ip & int(self.netmask), version=self._version) + self._cache['network'] = x + return x + + @property + def broadcast(self): + x = self._cache.get('broadcast') + if x is None: + x = IPAddress(self._ip | int(self.hostmask), version=self._version) + self._cache['broadcast'] = x + return x + + @property + def hostmask(self): + x = self._cache.get('hostmask') + if x is None: + x = IPAddress(int(self.netmask) ^ self._ALL_ONES, + version=self._version) + self._cache['hostmask'] = x + return x + + @property + def with_prefixlen(self): + return '%s/%d' % (str(self.ip), self._prefixlen) + + @property + def with_netmask(self): + return '%s/%s' % (str(self.ip), str(self.netmask)) + + @property + def with_hostmask(self): + return '%s/%s' % (str(self.ip), str(self.hostmask)) + + @property + def numhosts(self): + """Number of hosts in the current subnet.""" + return int(self.broadcast) - int(self.network) + 1 + + @property + def version(self): + raise NotImplementedError('BaseNet has no version') + + @property + def prefixlen(self): + return self._prefixlen + + def address_exclude(self, other): + """Remove an address from a larger block. + + For example: + + addr1 = IPNetwork('10.1.1.0/24') + addr2 = IPNetwork('10.1.1.0/26') + addr1.address_exclude(addr2) = + [IPNetwork('10.1.1.64/26'), IPNetwork('10.1.1.128/25')] + + or IPv6: + + addr1 = IPNetwork('::1/32') + addr2 = IPNetwork('::1/128') + addr1.address_exclude(addr2) = [IPNetwork('::0/128'), + IPNetwork('::2/127'), + IPNetwork('::4/126'), + IPNetwork('::8/125'), + ... + IPNetwork('0:0:8000::/33')] + + Args: + other: An IPvXNetwork object of the same type. + + Returns: + A sorted list of IPvXNetwork objects addresses which is self + minus other. + + Raises: + TypeError: If self and other are of difffering address + versions, or if other is not a network object. + ValueError: If other is not completely contained by self. + + """ + if not self._version == other._version: + raise TypeError("%s and %s are not of the same version" % ( + str(self), str(other))) + + if not isinstance(other, _BaseNet): + raise TypeError("%s is not a network object" % str(other)) + + if other not in self: + raise ValueError('%s not contained in %s' % (str(other), + str(self))) + if other == self: + return [] + + ret_addrs = [] + + # Make sure we're comparing the network of other. + other = IPNetwork('%s/%s' % (str(other.network), str(other.prefixlen)), + version=other._version) + + s1, s2 = self.subnet() + while s1 != other and s2 != other: + if other in s1: + ret_addrs.append(s2) + s1, s2 = s1.subnet() + elif other in s2: + ret_addrs.append(s1) + s1, s2 = s2.subnet() + else: + # If we got here, there's a bug somewhere. + assert True == False, ('Error performing exclusion: ' + 's1: %s s2: %s other: %s' % + (str(s1), str(s2), str(other))) + if s1 == other: + ret_addrs.append(s2) + elif s2 == other: + ret_addrs.append(s1) + else: + # If we got here, there's a bug somewhere. + assert True == False, ('Error performing exclusion: ' + 's1: %s s2: %s other: %s' % + (str(s1), str(s2), str(other))) + + return sorted(ret_addrs, key=_BaseNet._get_networks_key) + + def compare_networks(self, other): + """Compare two IP objects. + + This is only concerned about the comparison of the integer + representation of the network addresses. This means that the + host bits aren't considered at all in this method. If you want + to compare host bits, you can easily enough do a + 'HostA._ip < HostB._ip' + + Args: + other: An IP object. + + Returns: + If the IP versions of self and other are the same, returns: + + -1 if self < other: + eg: IPv4('1.1.1.0/24') < IPv4('1.1.2.0/24') + IPv6('1080::200C:417A') < IPv6('1080::200B:417B') + 0 if self == other + eg: IPv4('1.1.1.1/24') == IPv4('1.1.1.2/24') + IPv6('1080::200C:417A/96') == IPv6('1080::200C:417B/96') + 1 if self > other + eg: IPv4('1.1.1.0/24') > IPv4('1.1.0.0/24') + IPv6('1080::1:200C:417A/112') > + IPv6('1080::0:200C:417A/112') + + If the IP versions of self and other are different, returns: + + -1 if self._version < other._version + eg: IPv4('10.0.0.1/24') < IPv6('::1/128') + 1 if self._version > other._version + eg: IPv6('::1/128') > IPv4('255.255.255.0/24') + + """ + if self._version < other._version: + return -1 + if self._version > other._version: + return 1 + # self._version == other._version below here: + if self.network < other.network: + return -1 + if self.network > other.network: + return 1 + # self.network == other.network below here: + if self.netmask < other.netmask: + return -1 + if self.netmask > other.netmask: + return 1 + # self.network == other.network and self.netmask == other.netmask + return 0 + + def _get_networks_key(self): + """Network-only key function. + + Returns an object that identifies this address' network and + netmask. This function is a suitable "key" argument for sorted() + and list.sort(). + + """ + return (self._version, self.network, self.netmask) + + def _ip_int_from_prefix(self, prefixlen=None): + """Turn the prefix length netmask into a int for comparison. + + Args: + prefixlen: An integer, the prefix length. + + Returns: + An integer. + + """ + if not prefixlen and prefixlen != 0: + prefixlen = self._prefixlen + return self._ALL_ONES ^ (self._ALL_ONES >> prefixlen) + + def _prefix_from_ip_int(self, ip_int, mask=32): + """Return prefix length from the decimal netmask. + + Args: + ip_int: An integer, the IP address. + mask: The netmask. Defaults to 32. + + Returns: + An integer, the prefix length. + + """ + while mask: + if ip_int & 1 == 1: + break + ip_int >>= 1 + mask -= 1 + + return mask + + def _ip_string_from_prefix(self, prefixlen=None): + """Turn a prefix length into a dotted decimal string. + + Args: + prefixlen: An integer, the netmask prefix length. + + Returns: + A string, the dotted decimal netmask string. + + """ + if not prefixlen: + prefixlen = self._prefixlen + return self._string_from_ip_int(self._ip_int_from_prefix(prefixlen)) + + def iter_subnets(self, prefixlen_diff=1, new_prefix=None): + """The subnets which join to make the current subnet. + + In the case that self contains only one IP + (self._prefixlen == 32 for IPv4 or self._prefixlen == 128 + for IPv6), return a list with just ourself. + + Args: + prefixlen_diff: An integer, the amount the prefix length + should be increased by. This should not be set if + new_prefix is also set. + new_prefix: The desired new prefix length. This must be a + larger number (smaller prefix) than the existing prefix. + This should not be set if prefixlen_diff is also set. + + Returns: + An iterator of IPv(4|6) objects. + + Raises: + ValueError: The prefixlen_diff is too small or too large. + OR + prefixlen_diff and new_prefix are both set or new_prefix + is a smaller number than the current prefix (smaller + number means a larger network) + + """ + if self._prefixlen == self._max_prefixlen: + yield self + return + + if new_prefix is not None: + if new_prefix < self._prefixlen: + raise ValueError('new prefix must be longer') + if prefixlen_diff != 1: + raise ValueError('cannot set prefixlen_diff and new_prefix') + prefixlen_diff = new_prefix - self._prefixlen + + if prefixlen_diff < 0: + raise ValueError('prefix length diff must be > 0') + new_prefixlen = self._prefixlen + prefixlen_diff + + if not self._is_valid_netmask(str(new_prefixlen)): + raise ValueError( + 'prefix length diff %d is invalid for netblock %s' % ( + new_prefixlen, str(self))) + + first = IPNetwork('%s/%s' % (str(self.network), + str(self._prefixlen + prefixlen_diff)), + version=self._version) + + yield first + current = first + while True: + broadcast = current.broadcast + if broadcast == self.broadcast: + return + new_addr = IPAddress(int(broadcast) + 1, version=self._version) + current = IPNetwork('%s/%s' % (str(new_addr), str(new_prefixlen)), + version=self._version) + + yield current + + def masked(self): + """Return the network object with the host bits masked out.""" + return IPNetwork('%s/%d' % (self.network, self._prefixlen), + version=self._version) + + def subnet(self, prefixlen_diff=1, new_prefix=None): + """Return a list of subnets, rather than an iterator.""" + return list(self.iter_subnets(prefixlen_diff, new_prefix)) + + def supernet(self, prefixlen_diff=1, new_prefix=None): + """The supernet containing the current network. + + Args: + prefixlen_diff: An integer, the amount the prefix length of + the network should be decreased by. For example, given a + /24 network and a prefixlen_diff of 3, a supernet with a + /21 netmask is returned. + + Returns: + An IPv4 network object. + + Raises: + ValueError: If self.prefixlen - prefixlen_diff < 0. I.e., you have a + negative prefix length. + OR + If prefixlen_diff and new_prefix are both set or new_prefix is a + larger number than the current prefix (larger number means a + smaller network) + + """ + if self._prefixlen == 0: + return self + + if new_prefix is not None: + if new_prefix > self._prefixlen: + raise ValueError('new prefix must be shorter') + if prefixlen_diff != 1: + raise ValueError('cannot set prefixlen_diff and new_prefix') + prefixlen_diff = self._prefixlen - new_prefix + + + if self.prefixlen - prefixlen_diff < 0: + raise ValueError( + 'current prefixlen is %d, cannot have a prefixlen_diff of %d' % + (self.prefixlen, prefixlen_diff)) + return IPNetwork('%s/%s' % (str(self.network), + str(self.prefixlen - prefixlen_diff)), + version=self._version) + + # backwards compatibility + Subnet = subnet + Supernet = supernet + AddressExclude = address_exclude + CompareNetworks = compare_networks + Contains = __contains__ + + +class _BaseV4(object): + + """Base IPv4 object. + + The following methods are used by IPv4 objects in both single IP + addresses and networks. + + """ + + # Equivalent to 255.255.255.255 or 32 bits of 1's. + _ALL_ONES = (2**IPV4LENGTH) - 1 + _DECIMAL_DIGITS = frozenset('0123456789') + + def __init__(self, address): + self._version = 4 + self._max_prefixlen = IPV4LENGTH + + def _explode_shorthand_ip_string(self): + return str(self) + + def _ip_int_from_string(self, ip_str): + """Turn the given IP string into an integer for comparison. + + Args: + ip_str: A string, the IP ip_str. + + Returns: + The IP ip_str as an integer. + + Raises: + AddressValueError: if ip_str isn't a valid IPv4 Address. + + """ + octets = ip_str.split('.') + if len(octets) != 4: + raise AddressValueError(ip_str) + + packed_ip = 0 + for oc in octets: + try: + packed_ip = (packed_ip << 8) | self._parse_octet(oc) + except ValueError: + raise AddressValueError(ip_str) + return packed_ip + + def _parse_octet(self, octet_str): + """Convert a decimal octet into an integer. + + Args: + octet_str: A string, the number to parse. + + Returns: + The octet as an integer. + + Raises: + ValueError: if the octet isn't strictly a decimal from [0..255]. + + """ + # Whitelist the characters, since int() allows a lot of bizarre stuff. + if not self._DECIMAL_DIGITS.issuperset(octet_str): + raise ValueError + octet_int = int(octet_str, 10) + # Disallow leading zeroes, because no clear standard exists on + # whether these should be interpreted as decimal or octal. + if octet_int > 255 or (octet_str[0] == '0' and len(octet_str) > 1): + raise ValueError + return octet_int + + def _string_from_ip_int(self, ip_int): + """Turns a 32-bit integer into dotted decimal notation. + + Args: + ip_int: An integer, the IP address. + + Returns: + The IP address as a string in dotted decimal notation. + + """ + octets = [] + for _ in xrange(4): + octets.insert(0, str(ip_int & 0xFF)) + ip_int >>= 8 + return '.'.join(octets) + + @property + def max_prefixlen(self): + return self._max_prefixlen + + @property + def packed(self): + """The binary representation of this address.""" + return v4_int_to_packed(self._ip) + + @property + def version(self): + return self._version + + @property + def is_reserved(self): + """Test if the address is otherwise IETF reserved. + + Returns: + A boolean, True if the address is within the + reserved IPv4 Network range. + + """ + return self in IPv4Network('240.0.0.0/4') + + @property + def is_private(self): + """Test if this address is allocated for private networks. + + Returns: + A boolean, True if the address is reserved per RFC 1918. + + """ + return (self in IPv4Network('10.0.0.0/8') or + self in IPv4Network('172.16.0.0/12') or + self in IPv4Network('192.168.0.0/16')) + + @property + def is_multicast(self): + """Test if the address is reserved for multicast use. + + Returns: + A boolean, True if the address is multicast. + See RFC 3171 for details. + + """ + return self in IPv4Network('224.0.0.0/4') + + @property + def is_unspecified(self): + """Test if the address is unspecified. + + Returns: + A boolean, True if this is the unspecified address as defined in + RFC 5735 3. + + """ + return self in IPv4Network('0.0.0.0') + + @property + def is_loopback(self): + """Test if the address is a loopback address. + + Returns: + A boolean, True if the address is a loopback per RFC 3330. + + """ + return self in IPv4Network('127.0.0.0/8') + + @property + def is_link_local(self): + """Test if the address is reserved for link-local. + + Returns: + A boolean, True if the address is link-local per RFC 3927. + + """ + return self in IPv4Network('169.254.0.0/16') + + +class IPv4Address(_BaseV4, _BaseIP): + + """Represent and manipulate single IPv4 Addresses.""" + + def __init__(self, address): + + """ + Args: + address: A string or integer representing the IP + '192.168.1.1' + + Additionally, an integer can be passed, so + IPv4Address('192.168.1.1') == IPv4Address(3232235777). + or, more generally + IPv4Address(int(IPv4Address('192.168.1.1'))) == + IPv4Address('192.168.1.1') + + Raises: + AddressValueError: If ipaddr isn't a valid IPv4 address. + + """ + _BaseV4.__init__(self, address) + + # Efficient constructor from integer. + if isinstance(address, (int, long)): + self._ip = address + if address < 0 or address > self._ALL_ONES: + raise AddressValueError(address) + return + + # Constructing from a packed address + if isinstance(address, Bytes): + try: + self._ip, = struct.unpack('!I', address) + except struct.error: + raise AddressValueError(address) # Wrong length. + return + + # Assume input argument to be string or any object representation + # which converts into a formatted IP string. + addr_str = str(address) + self._ip = self._ip_int_from_string(addr_str) + + +class IPv4Network(_BaseV4, _BaseNet): + + """This class represents and manipulates 32-bit IPv4 networks. + + Attributes: [examples for IPv4Network('1.2.3.4/27')] + ._ip: 16909060 + .ip: IPv4Address('1.2.3.4') + .network: IPv4Address('1.2.3.0') + .hostmask: IPv4Address('0.0.0.31') + .broadcast: IPv4Address('1.2.3.31') + .netmask: IPv4Address('255.255.255.224') + .prefixlen: 27 + + """ + + # the valid octets for host and netmasks. only useful for IPv4. + _valid_mask_octets = set((255, 254, 252, 248, 240, 224, 192, 128, 0)) + + def __init__(self, address, strict=False): + """Instantiate a new IPv4 network object. + + Args: + address: A string or integer representing the IP [& network]. + '192.168.1.1/24' + '192.168.1.1/255.255.255.0' + '192.168.1.1/0.0.0.255' + are all functionally the same in IPv4. Similarly, + '192.168.1.1' + '192.168.1.1/255.255.255.255' + '192.168.1.1/32' + are also functionaly equivalent. That is to say, failing to + provide a subnetmask will create an object with a mask of /32. + + If the mask (portion after the / in the argument) is given in + dotted quad form, it is treated as a netmask if it starts with a + non-zero field (e.g. /255.0.0.0 == /8) and as a hostmask if it + starts with a zero field (e.g. 0.255.255.255 == /8), with the + single exception of an all-zero mask which is treated as a + netmask == /0. If no mask is given, a default of /32 is used. + + Additionally, an integer can be passed, so + IPv4Network('192.168.1.1') == IPv4Network(3232235777). + or, more generally + IPv4Network(int(IPv4Network('192.168.1.1'))) == + IPv4Network('192.168.1.1') + + strict: A boolean. If true, ensure that we have been passed + A true network address, eg, 192.168.1.0/24 and not an + IP address on a network, eg, 192.168.1.1/24. + + Raises: + AddressValueError: If ipaddr isn't a valid IPv4 address. + NetmaskValueError: If the netmask isn't valid for + an IPv4 address. + ValueError: If strict was True and a network address was not + supplied. + + """ + _BaseNet.__init__(self, address) + _BaseV4.__init__(self, address) + + # Constructing from an integer or packed bytes. + if isinstance(address, (int, long, Bytes)): + self.ip = IPv4Address(address) + self._ip = self.ip._ip + self._prefixlen = self._max_prefixlen + self.netmask = IPv4Address(self._ALL_ONES) + return + + # Assume input argument to be string or any object representation + # which converts into a formatted IP prefix string. + addr = str(address).split('/') + + if len(addr) > 2: + raise AddressValueError(address) + + self._ip = self._ip_int_from_string(addr[0]) + self.ip = IPv4Address(self._ip) + + if len(addr) == 2: + mask = addr[1].split('.') + if len(mask) == 4: + # We have dotted decimal netmask. + if self._is_valid_netmask(addr[1]): + self.netmask = IPv4Address(self._ip_int_from_string( + addr[1])) + elif self._is_hostmask(addr[1]): + self.netmask = IPv4Address( + self._ip_int_from_string(addr[1]) ^ self._ALL_ONES) + else: + raise NetmaskValueError('%s is not a valid netmask' + % addr[1]) + + self._prefixlen = self._prefix_from_ip_int(int(self.netmask)) + else: + # We have a netmask in prefix length form. + if not self._is_valid_netmask(addr[1]): + raise NetmaskValueError(addr[1]) + self._prefixlen = int(addr[1]) + self.netmask = IPv4Address(self._ip_int_from_prefix( + self._prefixlen)) + else: + self._prefixlen = self._max_prefixlen + self.netmask = IPv4Address(self._ip_int_from_prefix( + self._prefixlen)) + if strict: + if self.ip != self.network: + raise ValueError('%s has host bits set' % + self.ip) + if self._prefixlen == (self._max_prefixlen - 1): + self.iterhosts = self.__iter__ + + def _is_hostmask(self, ip_str): + """Test if the IP string is a hostmask (rather than a netmask). + + Args: + ip_str: A string, the potential hostmask. + + Returns: + A boolean, True if the IP string is a hostmask. + + """ + bits = ip_str.split('.') + try: + parts = [int(x) for x in bits if int(x) in self._valid_mask_octets] + except ValueError: + return False + if len(parts) != len(bits): + return False + if parts[0] < parts[-1]: + return True + return False + + def _is_valid_netmask(self, netmask): + """Verify that the netmask is valid. + + Args: + netmask: A string, either a prefix or dotted decimal + netmask. + + Returns: + A boolean, True if the prefix represents a valid IPv4 + netmask. + + """ + mask = netmask.split('.') + if len(mask) == 4: + if [x for x in mask if int(x) not in self._valid_mask_octets]: + return False + if [y for idx, y in enumerate(mask) if idx > 0 and + y > mask[idx - 1]]: + return False + return True + try: + netmask = int(netmask) + except ValueError: + return False + return 0 <= netmask <= self._max_prefixlen + + # backwards compatibility + IsRFC1918 = lambda self: self.is_private + IsMulticast = lambda self: self.is_multicast + IsLoopback = lambda self: self.is_loopback + IsLinkLocal = lambda self: self.is_link_local + + +class _BaseV6(object): + + """Base IPv6 object. + + The following methods are used by IPv6 objects in both single IP + addresses and networks. + + """ + + _ALL_ONES = (2**IPV6LENGTH) - 1 + _HEXTET_COUNT = 8 + _HEX_DIGITS = frozenset('0123456789ABCDEFabcdef') + + def __init__(self, address): + self._version = 6 + self._max_prefixlen = IPV6LENGTH + + def _ip_int_from_string(self, ip_str): + """Turn an IPv6 ip_str into an integer. + + Args: + ip_str: A string, the IPv6 ip_str. + + Returns: + A long, the IPv6 ip_str. + + Raises: + AddressValueError: if ip_str isn't a valid IPv6 Address. + + """ + parts = ip_str.split(':') + + # An IPv6 address needs at least 2 colons (3 parts). + if len(parts) < 3: + raise AddressValueError(ip_str) + + # If the address has an IPv4-style suffix, convert it to hexadecimal. + if '.' in parts[-1]: + ipv4_int = IPv4Address(parts.pop())._ip + parts.append('%x' % ((ipv4_int >> 16) & 0xFFFF)) + parts.append('%x' % (ipv4_int & 0xFFFF)) + + # An IPv6 address can't have more than 8 colons (9 parts). + if len(parts) > self._HEXTET_COUNT + 1: + raise AddressValueError(ip_str) + + # Disregarding the endpoints, find '::' with nothing in between. + # This indicates that a run of zeroes has been skipped. + try: + skip_index, = ( + [i for i in xrange(1, len(parts) - 1) if not parts[i]] or + [None]) + except ValueError: + # Can't have more than one '::' + raise AddressValueError(ip_str) + + # parts_hi is the number of parts to copy from above/before the '::' + # parts_lo is the number of parts to copy from below/after the '::' + if skip_index is not None: + # If we found a '::', then check if it also covers the endpoints. + parts_hi = skip_index + parts_lo = len(parts) - skip_index - 1 + if not parts[0]: + parts_hi -= 1 + if parts_hi: + raise AddressValueError(ip_str) # ^: requires ^:: + if not parts[-1]: + parts_lo -= 1 + if parts_lo: + raise AddressValueError(ip_str) # :$ requires ::$ + parts_skipped = self._HEXTET_COUNT - (parts_hi + parts_lo) + if parts_skipped < 1: + raise AddressValueError(ip_str) + else: + # Otherwise, allocate the entire address to parts_hi. The endpoints + # could still be empty, but _parse_hextet() will check for that. + if len(parts) != self._HEXTET_COUNT: + raise AddressValueError(ip_str) + parts_hi = len(parts) + parts_lo = 0 + parts_skipped = 0 + + try: + # Now, parse the hextets into a 128-bit integer. + ip_int = 0L + for i in xrange(parts_hi): + ip_int <<= 16 + ip_int |= self._parse_hextet(parts[i]) + ip_int <<= 16 * parts_skipped + for i in xrange(-parts_lo, 0): + ip_int <<= 16 + ip_int |= self._parse_hextet(parts[i]) + return ip_int + except ValueError: + raise AddressValueError(ip_str) + + def _parse_hextet(self, hextet_str): + """Convert an IPv6 hextet string into an integer. + + Args: + hextet_str: A string, the number to parse. + + Returns: + The hextet as an integer. + + Raises: + ValueError: if the input isn't strictly a hex number from [0..FFFF]. + + """ + # Whitelist the characters, since int() allows a lot of bizarre stuff. + if not self._HEX_DIGITS.issuperset(hextet_str): + raise ValueError + hextet_int = int(hextet_str, 16) + if hextet_int > 0xFFFF: + raise ValueError + return hextet_int + + def _compress_hextets(self, hextets): + """Compresses a list of hextets. + + Compresses a list of strings, replacing the longest continuous + sequence of "0" in the list with "" and adding empty strings at + the beginning or at the end of the string such that subsequently + calling ":".join(hextets) will produce the compressed version of + the IPv6 address. + + Args: + hextets: A list of strings, the hextets to compress. + + Returns: + A list of strings. + + """ + best_doublecolon_start = -1 + best_doublecolon_len = 0 + doublecolon_start = -1 + doublecolon_len = 0 + for index in range(len(hextets)): + if hextets[index] == '0': + doublecolon_len += 1 + if doublecolon_start == -1: + # Start of a sequence of zeros. + doublecolon_start = index + if doublecolon_len > best_doublecolon_len: + # This is the longest sequence of zeros so far. + best_doublecolon_len = doublecolon_len + best_doublecolon_start = doublecolon_start + else: + doublecolon_len = 0 + doublecolon_start = -1 + + if best_doublecolon_len > 1: + best_doublecolon_end = (best_doublecolon_start + + best_doublecolon_len) + # For zeros at the end of the address. + if best_doublecolon_end == len(hextets): + hextets += [''] + hextets[best_doublecolon_start:best_doublecolon_end] = [''] + # For zeros at the beginning of the address. + if best_doublecolon_start == 0: + hextets = [''] + hextets + + return hextets + + def _string_from_ip_int(self, ip_int=None): + """Turns a 128-bit integer into hexadecimal notation. + + Args: + ip_int: An integer, the IP address. + + Returns: + A string, the hexadecimal representation of the address. + + Raises: + ValueError: The address is bigger than 128 bits of all ones. + + """ + if not ip_int and ip_int != 0: + ip_int = int(self._ip) + + if ip_int > self._ALL_ONES: + raise ValueError('IPv6 address is too large') + + hex_str = '%032x' % ip_int + hextets = [] + for x in range(0, 32, 4): + hextets.append('%x' % int(hex_str[x:x+4], 16)) + + hextets = self._compress_hextets(hextets) + return ':'.join(hextets) + + def _explode_shorthand_ip_string(self): + """Expand a shortened IPv6 address. + + Args: + ip_str: A string, the IPv6 address. + + Returns: + A string, the expanded IPv6 address. + + """ + if isinstance(self, _BaseNet): + ip_str = str(self.ip) + else: + ip_str = str(self) + + ip_int = self._ip_int_from_string(ip_str) + parts = [] + for i in xrange(self._HEXTET_COUNT): + parts.append('%04x' % (ip_int & 0xFFFF)) + ip_int >>= 16 + parts.reverse() + if isinstance(self, _BaseNet): + return '%s/%d' % (':'.join(parts), self.prefixlen) + return ':'.join(parts) + + @property + def max_prefixlen(self): + return self._max_prefixlen + + @property + def packed(self): + """The binary representation of this address.""" + return v6_int_to_packed(self._ip) + + @property + def version(self): + return self._version + + @property + def is_multicast(self): + """Test if the address is reserved for multicast use. + + Returns: + A boolean, True if the address is a multicast address. + See RFC 2373 2.7 for details. + + """ + return self in IPv6Network('ff00::/8') + + @property + def is_reserved(self): + """Test if the address is otherwise IETF reserved. + + Returns: + A boolean, True if the address is within one of the + reserved IPv6 Network ranges. + + """ + return (self in IPv6Network('::/8') or + self in IPv6Network('100::/8') or + self in IPv6Network('200::/7') or + self in IPv6Network('400::/6') or + self in IPv6Network('800::/5') or + self in IPv6Network('1000::/4') or + self in IPv6Network('4000::/3') or + self in IPv6Network('6000::/3') or + self in IPv6Network('8000::/3') or + self in IPv6Network('A000::/3') or + self in IPv6Network('C000::/3') or + self in IPv6Network('E000::/4') or + self in IPv6Network('F000::/5') or + self in IPv6Network('F800::/6') or + self in IPv6Network('FE00::/9')) + + @property + def is_unspecified(self): + """Test if the address is unspecified. + + Returns: + A boolean, True if this is the unspecified address as defined in + RFC 2373 2.5.2. + + """ + return self._ip == 0 and getattr(self, '_prefixlen', 128) == 128 + + @property + def is_loopback(self): + """Test if the address is a loopback address. + + Returns: + A boolean, True if the address is a loopback address as defined in + RFC 2373 2.5.3. + + """ + return self._ip == 1 and getattr(self, '_prefixlen', 128) == 128 + + @property + def is_link_local(self): + """Test if the address is reserved for link-local. + + Returns: + A boolean, True if the address is reserved per RFC 4291. + + """ + return self in IPv6Network('fe80::/10') + + @property + def is_site_local(self): + """Test if the address is reserved for site-local. + + Note that the site-local address space has been deprecated by RFC 3879. + Use is_private to test if this address is in the space of unique local + addresses as defined by RFC 4193. + + Returns: + A boolean, True if the address is reserved per RFC 3513 2.5.6. + + """ + return self in IPv6Network('fec0::/10') + + @property + def is_private(self): + """Test if this address is allocated for private networks. + + Returns: + A boolean, True if the address is reserved per RFC 4193. + + """ + return self in IPv6Network('fc00::/7') + + @property + def ipv4_mapped(self): + """Return the IPv4 mapped address. + + Returns: + If the IPv6 address is a v4 mapped address, return the + IPv4 mapped address. Return None otherwise. + + """ + if (self._ip >> 32) != 0xFFFF: + return None + return IPv4Address(self._ip & 0xFFFFFFFF) + + @property + def teredo(self): + """Tuple of embedded teredo IPs. + + Returns: + Tuple of the (server, client) IPs or None if the address + doesn't appear to be a teredo address (doesn't start with + 2001::/32) + + """ + if (self._ip >> 96) != 0x20010000: + return None + return (IPv4Address((self._ip >> 64) & 0xFFFFFFFF), + IPv4Address(~self._ip & 0xFFFFFFFF)) + + @property + def sixtofour(self): + """Return the IPv4 6to4 embedded address. + + Returns: + The IPv4 6to4-embedded address if present or None if the + address doesn't appear to contain a 6to4 embedded address. + + """ + if (self._ip >> 112) != 0x2002: + return None + return IPv4Address((self._ip >> 80) & 0xFFFFFFFF) + + +class IPv6Address(_BaseV6, _BaseIP): + + """Represent and manipulate single IPv6 Addresses. + """ + + def __init__(self, address): + """Instantiate a new IPv6 address object. + + Args: + address: A string or integer representing the IP + + Additionally, an integer can be passed, so + IPv6Address('2001:4860::') == + IPv6Address(42541956101370907050197289607612071936L). + or, more generally + IPv6Address(IPv6Address('2001:4860::')._ip) == + IPv6Address('2001:4860::') + + Raises: + AddressValueError: If address isn't a valid IPv6 address. + + """ + _BaseV6.__init__(self, address) + + # Efficient constructor from integer. + if isinstance(address, (int, long)): + self._ip = address + if address < 0 or address > self._ALL_ONES: + raise AddressValueError(address) + return + + # Constructing from a packed address + if isinstance(address, Bytes): + try: + hi, lo = struct.unpack('!QQ', address) + except struct.error: + raise AddressValueError(address) # Wrong length. + self._ip = (hi << 64) | lo + return + + # Assume input argument to be string or any object representation + # which converts into a formatted IP string. + addr_str = str(address) + if not addr_str: + raise AddressValueError('') + + self._ip = self._ip_int_from_string(addr_str) + + +class IPv6Network(_BaseV6, _BaseNet): + + """This class represents and manipulates 128-bit IPv6 networks. + + Attributes: [examples for IPv6('2001:658:22A:CAFE:200::1/64')] + .ip: IPv6Address('2001:658:22a:cafe:200::1') + .network: IPv6Address('2001:658:22a:cafe::') + .hostmask: IPv6Address('::ffff:ffff:ffff:ffff') + .broadcast: IPv6Address('2001:658:22a:cafe:ffff:ffff:ffff:ffff') + .netmask: IPv6Address('ffff:ffff:ffff:ffff::') + .prefixlen: 64 + + """ + + + def __init__(self, address, strict=False): + """Instantiate a new IPv6 Network object. + + Args: + address: A string or integer representing the IPv6 network or the IP + and prefix/netmask. + '2001:4860::/128' + '2001:4860:0000:0000:0000:0000:0000:0000/128' + '2001:4860::' + are all functionally the same in IPv6. That is to say, + failing to provide a subnetmask will create an object with + a mask of /128. + + Additionally, an integer can be passed, so + IPv6Network('2001:4860::') == + IPv6Network(42541956101370907050197289607612071936L). + or, more generally + IPv6Network(IPv6Network('2001:4860::')._ip) == + IPv6Network('2001:4860::') + + strict: A boolean. If true, ensure that we have been passed + A true network address, eg, 192.168.1.0/24 and not an + IP address on a network, eg, 192.168.1.1/24. + + Raises: + AddressValueError: If address isn't a valid IPv6 address. + NetmaskValueError: If the netmask isn't valid for + an IPv6 address. + ValueError: If strict was True and a network address was not + supplied. + + """ + _BaseNet.__init__(self, address) + _BaseV6.__init__(self, address) + + # Constructing from an integer or packed bytes. + if isinstance(address, (int, long, Bytes)): + self.ip = IPv6Address(address) + self._ip = self.ip._ip + self._prefixlen = self._max_prefixlen + self.netmask = IPv6Address(self._ALL_ONES) + return + + # Assume input argument to be string or any object representation + # which converts into a formatted IP prefix string. + addr = str(address).split('/') + + if len(addr) > 2: + raise AddressValueError(address) + + self._ip = self._ip_int_from_string(addr[0]) + self.ip = IPv6Address(self._ip) + + if len(addr) == 2: + if self._is_valid_netmask(addr[1]): + self._prefixlen = int(addr[1]) + else: + raise NetmaskValueError(addr[1]) + else: + self._prefixlen = self._max_prefixlen + + self.netmask = IPv6Address(self._ip_int_from_prefix(self._prefixlen)) + + if strict: + if self.ip != self.network: + raise ValueError('%s has host bits set' % + self.ip) + if self._prefixlen == (self._max_prefixlen - 1): + self.iterhosts = self.__iter__ + + def _is_valid_netmask(self, prefixlen): + """Verify that the netmask/prefixlen is valid. + + Args: + prefixlen: A string, the netmask in prefix length format. + + Returns: + A boolean, True if the prefix represents a valid IPv6 + netmask. + + """ + try: + prefixlen = int(prefixlen) + except ValueError: + return False + return 0 <= prefixlen <= self._max_prefixlen + + @property + def with_netmask(self): + return self.with_prefixlen diff --git a/manifold/util/log.py b/manifold/util/log.py new file mode 100644 index 00000000..cdb187f1 --- /dev/null +++ b/manifold/util/log.py @@ -0,0 +1,288 @@ +import sys, logging, traceback, inspect, os.path +from logging import handlers +from manifold.util.singleton import Singleton +from manifold.util.options import Options +from manifold.util.misc import caller_name, make_list +from manifold.util import colors + +# TODO Log should take separately message strings and arguments to be able to +# remember which messages are seen several times, and also to allow for +# translation +# TODO How to log to stdout without putting None in self.log + +class Log(object): + __metaclass__ = Singleton + + DEFAULTS = { + # Logging + "rsyslog_enable" : False, + "rsyslog_host" : None, #"log.top-hat.info", + "rsyslog_port" : None, #28514, + "log_file" : "/var/log/manifold.log", + "log_level" : "DEBUG", + "debug" : "default", + "log_duplicates" : False + } + + # COLORS + color_ansi = { + 'DEBUG' : colors.MYGREEN, + 'INFO' : colors.MYBLUE, + 'WARNING': colors.MYWARNING, + 'ERROR' : colors.MYRED, + 'HEADER' : colors.MYHEADER, + 'END' : colors.MYEND, + 'RECORD' : colors.MYBLUE, + 'TMP' : colors.MYRED, + } + + @classmethod + def color(cls, color): + return cls.color_ansi[color] if color else '' + + # To remove duplicate messages + seen = {} + + def __init__(self, name='(default)'): + self.log = None # logging.getLogger(name) + self.files_to_keep = [] + self.init_log() + self.color = True + + + @classmethod + def init_options(self): + opt = Options() + + opt.add_option( + "--rsyslog-enable", action = "store_false", dest = "rsyslog_enable", + help = "Specify if log have to be written to a rsyslog server.", + default = self.DEFAULTS["rsyslog_enable"] + ) + opt.add_option( + "--rsyslog-host", dest = "rsyslog_host", + help = "Rsyslog hostname.", + default = self.DEFAULTS["rsyslog_host"] + ) + opt.add_option( + "--rsyslog-port", type = "int", dest = "rsyslog_port", + help = "Rsyslog port.", + default = self.DEFAULTS["rsyslog_port"] + ) + opt.add_option( + "-o", "--log-file", dest = "log_file", + help = "Log filename.", + default = self.DEFAULTS["log_file"] + ) + opt.add_option( + "-L", "--log-level", dest = "log_level", + choices = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + help = "Log level", + default = self.DEFAULTS["log_level"] + ) + opt.add_option( + "-d", "--debug", dest = "debug", + help = "Debug paths (a list of coma-separated python path: path.to.module.function).", + default = self.DEFAULTS["debug"] + ) + opt.add_option( + "", "--log_duplicates", action = "store_true", dest = "log_duplicates", + help = "Remove duplicate messages in logs", + default = self.DEFAULTS["log_duplicates"] + ) + + def init_log(self, options=object()): + # Initialize self.log (require self.files_to_keep) + if self.log: # for debugging by using stdout, log may be equal to None + if Options().rsyslog_host: + shandler = self.make_handler_rsyslog( + Options().rsyslog_host, + Options().rsyslog_port, + Options().log_level + ) + elif Options().log_file: + shandler = self.make_handler_locallog( + Options().log_file, + Options().log_level + ) + + #------------------------------------------------------------------------ + # Log + #------------------------------------------------------------------------ + + def make_handler_rsyslog(self, rsyslog_host, rsyslog_port, log_level): + """ + \brief (Internal usage) Prepare logging via rsyslog + \param rsyslog_host The hostname of the rsyslog server + \param rsyslog_port The port of the rsyslog server + \param log_level Log level + """ + # Prepare the handler + shandler = handlers.SysLogHandler( + (rsyslog_host, rsyslog_port), + facility = handlers.SysLogHandler.LOG_DAEMON + ) + + # The log file must remain open while daemonizing + self.prepare_handler(shandler, log_level) + return shandler + + def make_handler_locallog(self, log_filename, log_level): + """ + \brief (Internal usage) Prepare local logging + \param log_filename The file in which we write the logs + \param log_level Log level + """ + # Create directory in which we store the log file + log_dir = os.path.dirname(log_filename) + if log_dir and not os.path.exists(log_dir): + try: + os.makedirs(log_dir) + except OSError, why: + # XXX here we don't log since log is not initialized yet + print "OS error: %s" % why + + # Prepare the handler + shandler = logging.handlers.RotatingFileHandler( + log_filename, + backupCount = 0 + ) + + # The log file must remain open while daemonizing + self.files_to_keep.append(shandler.stream) + self.prepare_handler(shandler, log_level) + return shandler + + def prepare_handler(self, shandler, log_level): + """ + \brief (Internal usage) + \param shandler Handler used to log information + \param log_level Log level + """ + shandler.setLevel(log_level) + formatter = logging.Formatter("%(asctime)s: %(name)s: %(levelname)s %(message)s") + shandler.setFormatter(formatter) + self.log.addHandler(shandler) + self.log.setLevel(getattr(logging, log_level, logging.INFO)) + + def get_logger(self): + return self.log + + @classmethod + def print_msg(cls, msg, level=None, caller=None): + sys.stdout.write(cls.color(level)) + if level: + print "%s" % level, + if caller: + print "[%30s]" % caller, + print msg, + print cls.color('END') + + #--------------------------------------------------------------------- + # Log: logger abstraction + #--------------------------------------------------------------------- + + @classmethod + def build_message_string(cls, msg, ctx): + if ctx: + msg = [m % ctx for m in msg] + if isinstance(msg, (tuple, list)): + msg = map(lambda s : "%s" % s, msg) + msg = " ".join(msg) + else: + msg = "%s" % msg + return msg + + @classmethod + def log_message(cls, level, msg, ctx): + """ + \brief Logs an message + \param level (string) Log level + \param msg (string / list of strings) Message string, or List of message strings + \param ctx (dict) Context for the message strings + """ + caller = None + + if not Options().log_duplicates: + try: + count = cls.seen.get(msg, 0) + cls.seen[msg] = count + 1 + except TypeError, e: + # Unhashable types in msg + count = 0 + + if count == 1: + msg += (" -- REPEATED -- Future similar messages will be silently ignored. Please use the --log_duplicates option to allow for duplicates",) + elif count > 1: + return + + if level == 'DEBUG': + caller = caller_name(skip=3) + # Eventually remove "" added to the configuration file + try: + paths = tuple(s.strip(' \t\n\r') for s in Options().debug.split(',')) + except: + paths = None + if not paths or not caller.startswith(paths): + return + + logger = Log().get_logger() + msg_str = cls.build_message_string(msg, ctx) + + if logger: + logger_fct = getattr(logger, level.lower()) + logger_fct("%s(): %s" % (inspect.stack()[2][3], msg_str)) + else: + cls.print_msg(msg_str, level, caller) + + + @classmethod + def critical(cls, *msg, **ctx): + if not Options().log_level in ['CRITICAL']: + return + cls.log_message('CRITICAL', msg, ctx) + sys.exit(0) + + @classmethod + def error(cls, *msg, **ctx): + if not Options().log_level in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']: + return + cls.log_message('ERROR', msg, ctx) + logger = Log().get_logger() + if not Log().get_logger(): + traceback.print_exc() + sys.exit(0) + + @classmethod + def warning(cls, *msg, **ctx): + if not Options().log_level in ['DEBUG', 'INFO', 'WARNING']: + return + cls.log_message('WARNING', msg, ctx) + + @classmethod + def info(cls, *msg, **ctx): + if not Options().log_level in ['DEBUG', 'INFO']: + return + cls.log_message('INFO', msg, ctx) + + @classmethod + def debug(cls, *msg, **ctx): + if not Options().log_level in ['DEBUG']: + return + cls.log_message('DEBUG', msg, ctx) + + @classmethod + def tmp(cls, *msg): + cls.print_msg(' '.join(map(lambda x: "%r"%x, make_list(msg))), 'TMP', caller_name()) + + @classmethod + def record(cls, *msg): + #cls.print_msg(' '.join(map(lambda x: "%r"%x, make_list(msg))), 'RECORD', caller_name()) + pass + + @classmethod + def deprecated(cls, new): + #cls.print_msg("Function %s is deprecated, please use %s" % (caller_name(skip=3), new)) + pass + +Log.init_options() diff --git a/manifold/util/misc.py b/manifold/util/misc.py new file mode 100644 index 00000000..2ec3d029 --- /dev/null +++ b/manifold/util/misc.py @@ -0,0 +1,66 @@ +import os, glob, inspect +from types import StringTypes + +def find_local_modules(filepath): + modules = [] + for f in glob.glob(os.path.dirname(filepath)+"/*.py"): + name = os.path.basename(f)[:-3] + if name != '__init__': + modules.append(name) + return modules + +def make_list(elt): + if not elt or isinstance(elt, list): + return elt + if isinstance(elt, StringTypes): + return [elt] + if isinstance(elt, (tuple, set, frozenset)): + return list(elt) + + +# FROM: https://gist.github.com/techtonik/2151727 +# Public Domain, i.e. feel free to copy/paste +# Considered a hack in Python 2 + +import inspect + +def caller_name(skip=2): + """Get a name of a caller in the format module.class.method + + `skip` specifies how many levels of stack to skip while getting caller + name. skip=1 means "who calls me", skip=2 "who calls my caller" etc. + + An empty string is returned if skipped levels exceed stack height + """ + stack = inspect.stack() + start = 0 + skip + if len(stack) < start + 1: + return '' + parentframe = stack[start][0] + + name = [] + module = inspect.getmodule(parentframe) + # `modname` can be None when frame is executed directly in console + # TODO(techtonik): consider using __main__ + if module: + name.append(module.__name__) + # detect classname + if 'self' in parentframe.f_locals: + # I don't know any way to detect call from the object method + # XXX: there seems to be no way to detect static method call - it will + # be just a function call + name.append(parentframe.f_locals['self'].__class__.__name__) + codename = parentframe.f_code.co_name + if codename != '': # top level usually + name.append( codename ) # function or a method + del parentframe + return ".".join(name) + +def is_sublist(x, y, shortcut=None): + if not shortcut: shortcut = [] + if x == []: return (True, shortcut) + if y == []: return (False, None) + if x[0] == y[0]: + return is_sublist(x[1:],y[1:], shortcut) + else: + return is_sublist(x, y[1:], shortcut + [y[0]]) diff --git a/manifold/util/options.py b/manifold/util/options.py new file mode 100644 index 00000000..e09ad335 --- /dev/null +++ b/manifold/util/options.py @@ -0,0 +1,95 @@ +import sys +import os.path +import optparse +# xxx warning : this is not taken care of by the debian packaging +# cfgparse seems to be available by pip only (on debian, that is) +# there seems to be another package that might be used to do similar stuff +# python-configglue - Glues together optparse.OptionParser and ConfigParser.ConfigParser +# additionally argumentparser would probably be the way to go, notwithstanding +# xxx Moving this into the parse method so this module can at least be imported +#import cfgparse + +from manifold.util.singleton import Singleton + +# http://docs.python.org/dev/library/argparse.html#upgrading-optparse-code + +class Options(object): + + __metaclass__ = Singleton + + # We should be able to use another default conf file + CONF_FILE = '/etc/manifold.conf' + + def __init__(self, name = None): + self._opt = optparse.OptionParser() + self._defaults = {} + self._name = name + self.clear() + + def clear(self): + self.options = {} + self.add_option( + "-c", "--config", dest = "cfg_file", + help = "Config file to use.", + default = self.CONF_FILE + ) + self.uptodate = True + + def parse(self): + """ + \brief Parse options passed from command-line + """ + # add options here + + # if we have a logger singleton, add its options here too + # get defaults too + + # Initialize options to default values + import cfgparse + cfg = cfgparse.ConfigParser() + cfg.add_optparse_help_option(self._opt) + + # Load configuration file + try: + cfg_filename = sys.argv[sys.argv.index("-c") + 1] + try: + with open(cfg_filename): cfg.add_file(cfg_filename) + except IOError: + raise Exception, "Cannot open specified configuration file: %s" % cfg_filename + except ValueError: + try: + with open(self.CONF_FILE): cfg.add_file(self.CONF_FILE) + except IOError: pass + + for option_name in self._defaults: + cfg.add_option(option_name, default = self._defaults[option_name]) + + # Load/override options from configuration file and command-line + (options, args) = cfg.parse(self._opt) + self.options.update(vars(options)) + self.uptodate = True + + + def add_option(self, *args, **kwargs): + default = kwargs.get('default', None) + self._defaults[kwargs['dest']] = default + if 'default' in kwargs: + # This is very important otherwise file content is not taken into account + del kwargs['default'] + kwargs['help'] += " Defaults to %r." % default + self._opt.add_option(*args, **kwargs) + self.uptodate = False + + def get_name(self): + return self._name if self._name else os.path.basename(sys.argv[0]) + + def __repr__(self): + return "" % self.options + + def __getattr__(self, key): + if not self.uptodate: + self.parse() + return self.options.get(key, None) + + def __setattr(self, key, value): + self.options[key] = value diff --git a/manifold/util/plugin_factory.py b/manifold/util/plugin_factory.py new file mode 100644 index 00000000..1956355a --- /dev/null +++ b/manifold/util/plugin_factory.py @@ -0,0 +1,24 @@ +from manifold.util.log import Log + +class PluginFactory(type): + def __init__(cls, name, bases, dic): + #super(PluginFactory, cls).__init__(name, bases, dic) + type.__init__(cls, name, bases, dic) + + try: + registry = getattr(cls, 'registry') + except AttributeError: + setattr(cls, 'registry', {}) + registry = getattr(cls, 'registry') + # XXX + if name != "Gateway": + if name.endswith('Gateway'): + name = name[:-7] + name = name.lower() + registry[name] = cls + + def get(self, name): + return registry[name.lower()] + + # Adding a class method get to retrieve plugins by name + setattr(cls, 'get', classmethod(get)) diff --git a/manifold/util/predicate.py b/manifold/util/predicate.py new file mode 100644 index 00000000..fb32e4d8 --- /dev/null +++ b/manifold/util/predicate.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Class Predicate: +# Define a condition to join for example to Table instances. +# If this condition involves several fields, you may define a +# single Predicate using tuple of fields. +# +# Copyright (C) UPMC Paris Universitas +# Authors: +# Jordan Augé +# Marc-Olivier Buob + +from types import StringTypes +from manifold.util.type import returns, accepts + +from operator import ( + and_, or_, inv, add, mul, sub, mod, truediv, lt, le, ne, gt, ge, eq, neg +) + +# Define the inclusion operators +class contains(type): pass +class included(type): pass + +# New modifier: { contains +class Predicate: + + operators = { + '==' : eq, + '!=' : ne, + '<' : lt, + '<=' : le, + '>' : gt, + '>=' : ge, + '&&' : and_, + '||' : or_, + 'CONTAINS' : contains, + 'INCLUDED' : included + } + + operators_short = { + '=' : eq, + '~' : ne, + '<' : lt, + '[' : le, + '>' : gt, + ']' : ge, + '&' : and_, + '|' : or_, + '}' : contains, + '{' : included + } + + def __init__(self, *args, **kwargs): + """ + Build a Predicate instance. + Args: + kwargs: You can pass: + - 3 args (left, operator, right) + left: The left operand (it may be a String instance or a tuple) + operator: See Predicate.operators, this is the binary operator + involved in this Predicate. + right: The right value (it may be a String instance + or a literal (String, numerical value, tuple...)) + - 1 argument (list or tuple), containing three arguments + (variable, operator, value) + """ + if len(args) == 3: + key, op, value = args + elif len(args) == 1 and isinstance(args[0], (tuple,list)) and len(args[0]) == 3: + key, op, value = args[0] + elif len(args) == 1 and isinstance(args[0], Predicate): + key, op, value = args[0].get_tuple() + else: + raise Exception, "Bad initializer for Predicate (args = %r)" % args + + assert not isinstance(value, (frozenset, dict, set)), "Invalid value type (type = %r)" % type(value) + if isinstance(value, list): + value = tuple(value) + + self.key = key + if isinstance(op, StringTypes): + op = op.upper() + if op in self.operators.keys(): + self.op = self.operators[op] + elif op in self.operators_short.keys(): + self.op = self.operators_short[op] + else: + self.op = op + + if isinstance(value, list): + self.value = tuple(value) + else: + self.value = value + + @returns(StringTypes) + def __str__(self): + """ + Returns: + The '%s' representation of this Predicate. + """ + key, op, value = self.get_str_tuple() + if isinstance(value, (tuple, list, set, frozenset)): + value = [repr(v) for v in value] + value = "[%s]" % ", ".join(value) + return "%s %s %r" % (key, op, value) + + @returns(StringTypes) + def __repr__(self): + """ + Returns: + The '%r' representation of this Predicate. + """ + return "Predicate<%s %s %r>" % self.get_str_tuple() + + def __hash__(self): + """ + Returns: + The hash of this Predicate (this allows to define set of + Predicate instances). + """ + return hash(self.get_tuple()) + + @returns(bool) + def __eq__(self, predicate): + """ + Returns: + True iif self == predicate. + """ + if not predicate: + return False + return self.get_tuple() == predicate.get_tuple() + + def get_key(self): + """ + Returns: + The left operand of this Predicate. It may be a String + or a tuple of Strings. + """ + return self.key + + def set_key(self, key): + """ + Set the left operand of this Predicate. + Params: + key: The new left operand. + """ + self.key = key + + def get_op(self): + return self.op + + def get_value(self): + return self.value + + def set_value(self, value): + self.value = value + + def get_tuple(self): + return (self.key, self.op, self.value) + + def get_str_op(self): + op_str = [s for s, op in self.operators.iteritems() if op == self.op] + return op_str[0] + + def get_str_tuple(self): + return (self.key, self.get_str_op(), self.value,) + + def to_list(self): + return list(self.get_str_tuple()) + + def match(self, dic, ignore_missing=False): + if isinstance(self.key, tuple): + print "PREDICATE MATCH", self.key + print dic + print "-----------------------------" + + # Can we match ? + if self.key not in dic: + return ignore_missing + + if self.op == eq: + if isinstance(self.value, list): + return (dic[self.key] in self.value) # array ? + else: + return (dic[self.key] == self.value) + elif self.op == ne: + if isinstance(self.value, list): + return (dic[self.key] not in self.value) # array ? + else: + return (dic[self.key] != self.value) # array ? + elif self.op == lt: + if isinstance(self.value, StringTypes): + # prefix match + return dic[self.key].startswith('%s.' % self.value) + else: + return (dic[self.key] < self.value) + elif self.op == le: + if isinstance(self.value, StringTypes): + return dic[self.key] == self.value or dic[self.key].startswith('%s.' % self.value) + else: + return (dic[self.key] <= self.value) + elif self.op == gt: + if isinstance(self.value, StringTypes): + # prefix match + return self.value.startswith('%s.' % dic[self.key]) + else: + return (dic[self.key] > self.value) + elif self.op == ge: + if isinstance(self.value, StringTypes): + # prefix match + return dic[self.key] == self.value or self.value.startswith('%s.' % dic[self.key]) + else: + return (dic[self.key] >= self.value) + elif self.op == and_: + return (dic[self.key] & self.value) # array ? + elif self.op == or_: + return (dic[self.key] | self.value) # array ? + elif self.op == contains: + method, subfield = self.key.split('.', 1) + return not not [ x for x in dic[method] if x[subfield] == self.value] + elif self.op == included: + return dic[self.key] in self.value + else: + raise Exception, "Unexpected table format: %r" % dic + + def filter(self, dic): + """ + Filter dic according to the current predicate. + """ + + if '.' in self.key: + # users.hrn + method, subfield = self.key.split('.', 1) + if not method in dic: + return None # XXX + + if isinstance(dic[method], dict): + # We have a 1..1 relationship: apply the same filter to the dict + subpred = Predicate(subfield, self.op, self.value) + match = subpred.match(dic[method]) + return dic if match else None + + elif isinstance(dic[method], (list, tuple)): + # 1..N relationships + match = False + if self.op == contains: + return dic if self.match(dic) else None + else: + subpred = Predicate(subfield, self.op, self.value) + dic[method] = subpred.filter(dic[method]) + return dic + else: + raise Exception, "Unexpected table format: %r", dic + + + else: + # Individual field operations: this could be simplified, since we are now using operators_short !! + # XXX match + print "current predicate", self + print "matching", dic + print "----" + return dic if self.match(dic) else None + + def get_field_names(self): + if isinstance(self.key, (list, tuple, set, frozenset)): + return set(self.key) + else: + return set([self.key]) + + def get_value_names(self): + if isinstance(self.value, (list, tuple, set, frozenset)): + return set(self.value) + else: + return set([self.value]) + + def has_empty_value(self): + if isinstance(self.value, (list, tuple, set, frozenset)): + return not any(self.value) + else: + return not self.value diff --git a/manifold/util/reactor_thread.py b/manifold/util/reactor_thread.py new file mode 100644 index 00000000..4eb2c7e2 --- /dev/null +++ b/manifold/util/reactor_thread.py @@ -0,0 +1,103 @@ +# Borrowed from Chandler +# http://chandlerproject.org/Projects/ChandlerTwistedInThreadedEnvironment + +import threading, time +from manifold.util.singleton import Singleton +from manifold.util.log import * +from twisted.internet import defer +from twisted.python import threadable + +__author__ ="Brian Kirsch " + +#required for using threads with the Reactor +threadable.init() + +class ReactorException(Exception): + def __init__(self, *args): + Exception.__init__(self, *args) + + +class ReactorThread(threading.Thread): + """ + Run the Reactor in a Thread to prevent blocking the + Main Thread once reactor.run is called + """ + + __metaclass__ = Singleton + + def __init__(self): + threading.Thread.__init__(self) + self._reactorRunning = False + + # Be sure the import is done only at runtime, we keep a reference in the + # class instance + from twisted.internet import reactor + self.reactor = reactor + + def run(self): + if self._reactorRunning: + raise ReactorException("Reactor Already Running") + + self._reactorRunning = True + + #call run passing a False flag indicating to the + #reactor not to install sig handlers since sig handlers + #only work on the main thread + try: + #signal.signal(signal.SIGINT, signal.default_int_handler) + self.reactor.run(False) + except Exception, e: + print "Reactor exception:", e + + def callInReactor(self, callable, *args, **kw): + if self._reactorRunning: + self.reactor.callFromThread(callable, *args, **kw) + else: + callable(*args, **kw) + + def isReactorRunning(self): + return self._reactorRunning + + def start_reactor(self): + if self._reactorRunning: + log_warning("Reactor already running. This is normal, please remove this debug message") + return + #raise ReactorException("Reactor Already Running") + threading.Thread.start(self) + cpt = 0 + while not self._reactorRunning: + time.sleep(0.1) + cpt +=1 + if cpt > 5: + raise ReactorException, "Reactor thread is too long to start... cancelling" + self.reactor.addSystemEventTrigger('after', 'shutdown', self.__reactorShutDown) + + def stop_reactor(self): + """ + may want a way to force thread to join if reactor does not shutdown + properly. The reactor can get in to a recursive loop condition if reactor.stop + placed in the threads join method. This will require further investigation. + """ + if not self._reactorRunning: + raise ReactorException("Reactor Not Running") + self.reactor.callFromThread(self.reactor.stop) + #self.reactor.join() + + def addReactorEventTrigger(self, phase, eventType, callable): + if self._reactorRunning: + self.reactor.callFromThread(self.reactor.addSystemEventTrigger, phase, eventType, callable) + else: + self.reactor.addSystemEventTrigger(phase, eventType, callable) + + def __reactorShuttingDown(self): + pass + + def __reactorShutDown(self): + """This method called when the reactor is stopped""" + self._reactorRunning = False + + def __getattr__(self, name): + # We transfer missing methods to the reactor + def _missing(*args, **kwargs): + self.reactor.callFromThread(getattr(self.reactor, name), *args, **kwargs) + return _missing diff --git a/manifold/util/reactor_wrapper.py b/manifold/util/reactor_wrapper.py new file mode 100644 index 00000000..eb18874f --- /dev/null +++ b/manifold/util/reactor_wrapper.py @@ -0,0 +1,48 @@ +from manifold.util.singleton import Singleton + +class ReactorWrapper(object): + __metaclass__ = Singleton + + def __init__(self): + # Be sure the import is done only at runtime, we keep a reference in the + # class instance + from twisted.internet import reactor + self.reactor = reactor + + + def callInReactor(self, callable, *args, **kw): + print "ReactorWrapper::callInReactor" + if self._reactorRunning: + self.reactor.callFromThread(callable, *args, **kw) + else: + callable(*args, **kw) + + def isReactorRunning(self): + return self._reactorRunning + + def start_reactor(self): + self.reactor.run() + + def stop_reactor(self): + self.reactor.stop() + + def addReactorEventTrigger(self, phase, eventType, callable): + print "ReactorWrapper::addReactorEventTrigger" + if self._reactorRunning: + self.reactor.callFromThread(self.reactor.addSystemEventTrigger, phase, eventType, callable) + else: + self.reactor.addSystemEventTrigger(phase, eventType, callable) + + def __reactorShuttingDown(self): + pass + + def __reactorShutDown(self): + """This method called when the reactor is stopped""" + print "REACTOR SHUTDOWN" + self._reactorRunning = False + + def __getattr__(self, name): + # We transfer missing methods to the reactor + def _missing(*args, **kwargs): + getattr(self.reactor, name)(*args, **kwargs) + return _missing diff --git a/manifold/util/singleton.py b/manifold/util/singleton.py new file mode 100644 index 00000000..b622c135 --- /dev/null +++ b/manifold/util/singleton.py @@ -0,0 +1,19 @@ +#------------------------------------------------------------------------- +# Class Singleton +# +# Classes that inherit from Singleton can be instanciated only once +#------------------------------------------------------------------------- + +class Singleton(type): + def __init__(cls, name, bases, dic): + super(Singleton,cls).__init__(name,bases,dic) + cls.instance=None + + def __call__(cls, *args, **kw): + if cls.instance is None: + cls.instance=super(Singleton,cls).__call__(*args,**kw) + return cls.instance + + +# See also +# http://stackoverflow.com/questions/6760685/creating-a-singleton-in-python diff --git a/manifold/util/storage.py b/manifold/util/storage.py new file mode 100644 index 00000000..066993e6 --- /dev/null +++ b/manifold/util/storage.py @@ -0,0 +1,29 @@ +from manifold.gateways import Gateway +from manifold.util.callback import Callback + +#URL='sqlite:///:memory:?check_same_thread=False' +URL='sqlite:////var/myslice/db.sqlite?check_same_thread=False' + +class Storage(object): + pass + # We can read information from files, database, commandline, etc + # Let's focus on the database + + @classmethod + def register(self, object): + """ + Registers a new object that will be stored locally by manifold. + This will live in the + """ + pass + +class DBStorage(Storage): + @classmethod + def execute(self, query, user=None, format='dict'): + # XXX Need to pass local parameters + gw = Gateway.get('sqlalchemy')(config={'url': URL}, user=user, format=format) + gw.set_query(query) + cb = Callback() + gw.set_callback(cb) + gw.start() + return cb.get_results() diff --git a/manifold/util/type.py b/manifold/util/type.py new file mode 100644 index 00000000..1cc03b2b --- /dev/null +++ b/manifold/util/type.py @@ -0,0 +1,144 @@ +# http://wiki.python.org/moin/PythonDecoratorLibrary#Type_Enforcement_.28accepts.2Freturns.29 +''' +One of three degrees of enforcement may be specified by passing +the 'debug' keyword argument to the decorator: + 0 -- NONE: No type-checking. Decorators disabled. + 1 -- MEDIUM: Print warning message to stderr. (Default) + 2 -- STRONG: Raise TypeError with message. +If 'debug' is not passed to the decorator, the default level is used. + +Example usage: + >>> NONE, MEDIUM, STRONG = 0, 1, 2 + >>> + >>> @accepts(int, int, int) + ... @returns(float) + ... def average(x, y, z): + ... return (x + y + z) / 2 + ... + >>> average(5.5, 10, 15.0) + TypeWarning: 'average' method accepts (int, int, int), but was given + (float, int, float) + 15.25 + >>> average(5, 10, 15) + TypeWarning: 'average' method returns (float), but result is (int) + 15 + +Needed to cast params as floats in function def (or simply divide by 2.0). + + >>> TYPE_CHECK = STRONG + >>> @accepts(int, debug=TYPE_CHECK) + ... @returns(int, debug=TYPE_CHECK) + ... def fib(n): + ... if n in (0, 1): return n + ... return fib(n-1) + fib(n-2) + ... + >>> fib(5.3) + Traceback (most recent call last): + ... + TypeError: 'fib' method accepts (int), but was given (float) + +''' +import sys +from itertools import izip + +def accepts(*types, **kw): + '''Function decorator. Checks decorated function's arguments are + of the expected types. + + Parameters: + types -- The expected types of the inputs to the decorated function. + Must specify type for each parameter. + kw -- Optional specification of 'debug' level (this is the only valid + keyword argument, no other should be given). + debug = ( 0 | 1 | 2 ) + + ''' + if not kw: + # default level: MEDIUM + debug = 2 + else: + debug = kw['debug'] + try: + def decorator(f): + # XXX Missing full support of kwargs + def newf(*args, **kwargs): + if debug is 0: + return f(*args, **kwargs) + assert len(args) == len(types) + argtypes = tuple(map(type, args)) + if not compare_types(types, argtypes): + # if argtypes != types: + msg = info(f.__name__, types, argtypes, 0) + if debug is 1: + print >> sys.stderr, 'TypeWarning: ', msg + elif debug is 2: + raise TypeError, msg + return f(*args, **kwargs) + newf.__name__ = f.__name__ + return newf + return decorator + except KeyError, key: + raise KeyError, key + "is not a valid keyword argument" + except TypeError, msg: + raise TypeError, msg + +def compare_types(expected, actual): + if isinstance(expected, tuple): + if isinstance(actual, tuple): + for x, y in izip(expected, actual): + if not compare_types(x ,y): + return False + return True + else: + return actual == type(None) or actual in expected + else: + return actual == type(None) or actual == expected or isinstance(actual, expected) # issubclass(actual, expected) + +def returns(ret_type, **kw): + '''Function decorator. Checks decorated function's return value + is of the expected type. + + Parameters: + ret_type -- The expected type of the decorated function's return value. + Must specify type for each parameter. + kw -- Optional specification of 'debug' level (this is the only valid + keyword argument, no other should be given). + debug=(0 | 1 | 2) + ''' + try: + if not kw: + # default level: MEDIUM + debug = 1 + else: + debug = kw['debug'] + def decorator(f): + def newf(*args): + result = f(*args) + if debug is 0: + return result + res_type = type(result) + if not compare_types(ret_type, res_type): + # if res_type != ret_type: # JORDAN: fix to allow for # StringTypes = (str, unicode) + # XXX note that this check should be recursive + msg = info(f.__name__, (ret_type,), (res_type,), 1) + if debug is 1: + print >> sys.stderr, 'TypeWarning: ', msg + elif debug is 2: + raise TypeError, msg + return result + newf.__name__ = f.__name__ + return newf + return decorator + except KeyError, key: + raise KeyError, key + "is not a valid keyword argument" + except TypeError, msg: + raise TypeError, msg + +def info(fname, expected, actual, flag): + '''Convenience function returns nicely formatted error/warning msg.''' + format = lambda types: ', '.join([str(t).split("'")[1] for t in types]) + msg = "'{}' method ".format( fname )\ + + ("accepts", "returns")[flag] + " ({}), but ".format(expected)\ + + ("was given", "result is")[flag] + " ({})".format(actual) + return msg + diff --git a/manifold/util/xmldict.py b/manifold/util/xmldict.py new file mode 100644 index 00000000..e45af734 --- /dev/null +++ b/manifold/util/xmldict.py @@ -0,0 +1,77 @@ +import os +import xml.etree.cElementTree as ElementTree + +class XmlListConfig(list): + def __init__(self, aList): + for element in aList: + if element: + # treat like dict + if len(element) == 1 or element[0].tag != element[1].tag: + self.append(XmlDictConfig(element)) + # treat like list + elif element[0].tag == element[1].tag: + self.append(XmlListConfig(element)) + elif element.text: + text = element.text.strip() + if text: + self.append(text) + + +class XmlDictConfig(dict): + ''' + Example usage: + + >>> tree = ElementTree.parse('your_file.xml') + >>> root = tree.getroot() + >>> xmldict = XmlDictConfig(root) + + Or, if you want to use an XML string: + + >>> root = ElementTree.XML(xml_string) + >>> xmldict = XmlDictConfig(root) + + And then use xmldict for what it is... a dict. + ''' + def __init__(self, parent_element): + childrenNames = [child.tag for child in parent_element.getchildren()] + + if parent_element.items(): #attributes + self.update(dict(parent_element.items())) + for element in parent_element: + if element: + # treat like dict - we assume that if the first two tags + # in a series are different, then they are all different. + if len(element) == 1 or element[0].tag != element[1].tag: + aDict = XmlDictConfig(element) + # treat like list - we assume that if the first two tags + # in a series are the same, then the rest are the same. + else: + # here, we put the list in dictionary; the key is the + # tag name the list elements all share in common, and + # the value is the list itself + aDict = {element[0].tag: XmlListConfig(element)} + # if the tag has attributes, add those to the dict + if element.items(): + aDict.update(dict(element.items())) + + if childrenNames.count(element.tag) > 1: + try: + currentValue = self[element.tag] + currentValue.append(aDict) + self.update({element.tag: currentValue}) + except: #the first of its kind, an empty list must be created + self.update({element.tag: [aDict]}) #aDict is written in [], i.e. it will be a list + + else: + self.update({element.tag: aDict}) + # this assumes that if you've got an attribute in a tag, + # you won't be having any text. This may or may not be a + # good idea -- time will tell. It works for the way we are + # currently doing XML configuration files... + elif element.items(): + self.update({element.tag: dict(element.items())}) + # finally, if there are no child tags and no attributes, extract + # the text + else: + self.update({element.tag: element.text}) + diff --git a/myslice/settings.py b/myslice/settings.py index 70740e3d..235a18ca 100644 --- a/myslice/settings.py +++ b/myslice/settings.py @@ -236,6 +236,8 @@ INSTALLED_APPS = [ # Uncomment the next line to enable admin documentation: # 'django.contrib.admindocs', 'portal', + # SLA + 'sla', ] # this app won't load in a build environment if not building: INSTALLED_APPS.append ('rest') @@ -287,3 +289,9 @@ CSRF_FAILURE_VIEW = 'manifoldapi.manifoldproxy.csrf_failure' #IA_JS_FORMAT = " -{% endif %} \ No newline at end of file +{% endif %} diff --git a/portal/templates/fed4fire/fed4fire_widget-slice-sections.html b/portal/templates/fed4fire/fed4fire_widget-slice-sections.html index bf64b8d8..fb9e64bb 100644 --- a/portal/templates/fed4fire/fed4fire_widget-slice-sections.html +++ b/portal/templates/fed4fire/fed4fire_widget-slice-sections.html @@ -7,6 +7,7 @@
  • Statistics
  • Measurements
  • Experiment
    • +
  • SLA
    • {% else %} -{% endif %} \ No newline at end of file +{% endif %} diff --git a/portal/templates/fed4fire/fed4fire_widget-topmenu.html b/portal/templates/fed4fire/fed4fire_widget-topmenu.html index 74edfdc9..173d87e3 100644 --- a/portal/templates/fed4fire/fed4fire_widget-topmenu.html +++ b/portal/templates/fed4fire/fed4fire_widget-topmenu.html @@ -40,6 +40,7 @@ +
  • |
    • diff --git a/portal/templates/servicedirectory.html b/portal/templates/servicedirectory.html new file mode 100644 index 00000000..7a1893f6 --- /dev/null +++ b/portal/templates/servicedirectory.html @@ -0,0 +1,254 @@ +{% extends "layout_wide.html" %} + +{% block head %} + +{% endblock head %} + +{% block content %} +
    +
    +
    + +
    +
    +
    +
    +
    +
    +
    Loading Services
    + +
    +
    + +
    +
    +
    Loading Services
    + +
    +
    +
    + + +{% endblock %} diff --git a/portal/templates/slice-resource-view.html b/portal/templates/slice-resource-view.html index 707fa364..39b04ce3 100644 --- a/portal/templates/slice-resource-view.html +++ b/portal/templates/slice-resource-view.html @@ -83,6 +83,9 @@ + {% endblock %} diff --git a/portal/urls.py b/portal/urls.py index c055685d..157c6dd3 100644 --- a/portal/urls.py +++ b/portal/urls.py @@ -45,6 +45,8 @@ from portal.validationview import ValidatePendingView from portal.univbrisview import UnivbrisView +from portal.servicedirectory import ServiceDirectoryView + from portal.documentationview import DocumentationView from portal.supportview import SupportView from portal.emailactivationview import ActivateEmailView @@ -117,7 +119,8 @@ urlpatterns = patterns('', 'portal.django_passresetview.password_reset_complete'), url(r'^univbris/?$', UnivbrisView.as_view(), name='univbris'), - # ... + + url(r'^servicedirectory/?$', ServiceDirectoryView.as_view(), name='servicedirectory'), ) # (r'^accounts/', include('registration.backends.default.urls')), diff --git a/sample/dashboardview.py b/sample/dashboardview.py index 1f68cc66..994164dd 100644 --- a/sample/dashboardview.py +++ b/sample/dashboardview.py @@ -15,7 +15,7 @@ from plugins.lists.slicelist import SliceList from plugins.querycode import QueryCode from plugins.quickfilter import QuickFilter -from trash.trashutils import quickfilter_criterias +from trashutils import quickfilter_criterias # from ui.topmenu import topmenu_items_live, the_user diff --git a/sample/pluginview.py b/sample/pluginview.py index 144115e9..bef0768f 100644 --- a/sample/pluginview.py +++ b/sample/pluginview.py @@ -20,7 +20,7 @@ from plugins.messages import Messages from plugins.querytable import QueryTable from ui.topmenu import topmenu_items_live, the_user -from trash.trashutils import hard_wired_slice_names, hard_wired_list, lorem_p, lorem, quickfilter_criterias +from trashutils import hard_wired_slice_names, hard_wired_list, lorem_p, lorem, quickfilter_criterias #might be useful or not depending on the context #@login_required diff --git a/sample/scrollview.py b/sample/scrollview.py index 605cb3cd..0ad69043 100644 --- a/sample/scrollview.py +++ b/sample/scrollview.py @@ -7,7 +7,7 @@ from unfold.prelude import Prelude from ui.topmenu import topmenu_items, the_user # tmp -from trash.trashutils import lorem, hard_wired_slice_names +from trashutils import lorem, hard_wired_slice_names def scroll_view (request): return render_to_response ('view-scroll.html', diff --git a/sample/tabview.py b/sample/tabview.py index 95c48286..641e4916 100644 --- a/sample/tabview.py +++ b/sample/tabview.py @@ -7,7 +7,7 @@ from unfold.prelude import Prelude from ui.topmenu import topmenu_items, the_user # tmp -from trash.trashutils import lorem, hard_wired_slice_names +from trashutils import lorem, hard_wired_slice_names @login_required def tab_view (request): diff --git a/sla/README b/sla/README new file mode 100755 index 00000000..3ecd9d96 --- /dev/null +++ b/sla/README @@ -0,0 +1,91 @@ +This is the README.txt file for sla-dashboard application. + +sla-dashboard application is composed by the following directories: +* sladashboard: the app related to the application itself. The settings + file maybe need to be modified: read below. +* slagui: the sla dashboard GUI project. +* slaclient: this project contains all the code needed to connect to + SLA Manager REST interface, and the conversion from xml/json to python + objects. +* samples: this directory contains sample files to load in the SLA Manager for + testing. +* bin: some useful scripts + + +Software requirements +--------------------- +Python version: 2.7.x + +The required python packages are listed in requirements.txt + +Installing the requirements inside a virtualenv is recommended. + +SLA Manager (java backend) needs to be running in order to use the dashboard. + +Installing +---------- +(to be corrected/completed) + +# +# Install virtualenv +# +$ pip install virtualenv + + +# +# Create virtualenv. +# E.g.: VIRTUALENVS_DIR=~/virtualenvs +# +$ virtualenv $VIRTUALENVS_DIR/sla-dashboard + +# +# Activate virtualenv +# +$ . $VIRTUALENVS_DIR/sla-dashboard/bin/activate + +# +# Change to application dir and install requirements +# +$ cd $SLA_DASHBOARD +$ pip install -r requirements.txt + +# +# Create needed tables for sessions, admin, etc +# +$ ./manage.py syncdb + +Settings +-------- + +* sladashboard/settings.py: + - SLA_MANAGER_URL : The URL of the SLA Manager REST interface. + - DEBUG: Please, set this to FALSE in production + +* sladashboard/urls.py: + - dashboard root url: the slagui project is accessed by default + in $server:$port/slagui. Change "slagui" with the desired path. + + +Running +------- +NOTE: this steps are not suitable in production mode. + +# +# Activate virtualenv +# +$ . $VIRTUALENVS_DIR/sla-dashboard/bin/activate + +# +# Cd to application dir +# +$ cd $SLA_DASHBOARD + +# +# Start server listing in port 8000 (change port as desired) +# +$ ./manage.py runserver 0.0.0.0:8000 + +# +# Test +# +curl http://localhost:8000/slagui \ No newline at end of file diff --git a/sla/__init__.py b/sla/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/sla/requirements.txt b/sla/requirements.txt new file mode 100755 index 00000000..277251d5 --- /dev/null +++ b/sla/requirements.txt @@ -0,0 +1,6 @@ +Django==1.5.2 +django-extensions +south +django-debug-toolbar +requests +python-dateutil<2.0 diff --git a/sla/samples/TemplateIMindsService.xml b/sla/samples/TemplateIMindsService.xml new file mode 100755 index 00000000..2faccaf5 --- /dev/null +++ b/sla/samples/TemplateIMindsService.xml @@ -0,0 +1,55 @@ + + + Template for iMinds service + + iMinds + AgreementInitiator + 2015-03-07T12:00:00:000 + iMinds service + + + + + + The template for iMinds service + + + + + + + + + iMinds/UpTime + + + iMinds/Performance + + + + + + + + UpTime + + {"constraint" : "UpTime GT 75"} + + + + + + + + + Performance + + {"constraint" : "Performance GT 50"} + + + + + + + diff --git a/sla/samples/providerCreate.bat b/sla/samples/providerCreate.bat new file mode 100755 index 00000000..58354f6a --- /dev/null +++ b/sla/samples/providerCreate.bat @@ -0,0 +1 @@ +curl -u normal_user:password -H "Content-type: application/xml" -d@providerIMinds.xml localhost:8080/sla-service/providers -X POST \ No newline at end of file diff --git a/sla/samples/providerIMinds.xml b/sla/samples/providerIMinds.xml new file mode 100755 index 00000000..1f81ce15 --- /dev/null +++ b/sla/samples/providerIMinds.xml @@ -0,0 +1,5 @@ + + + iMinds + iMinds Testbed + diff --git a/sla/samples/simpleAgreementCreation.bat b/sla/samples/simpleAgreementCreation.bat new file mode 100755 index 00000000..3a35b06f --- /dev/null +++ b/sla/samples/simpleAgreementCreation.bat @@ -0,0 +1 @@ +curl -Umyuser:mypassword -H "Content-type: application/json" -d@simpleAgreementCreationParameters.json localhost:8000/sla/agreements/simplecreate -X POST \ No newline at end of file diff --git a/sla/samples/simpleAgreementCreationParameters.json b/sla/samples/simpleAgreementCreationParameters.json new file mode 100755 index 00000000..c013f738 --- /dev/null +++ b/sla/samples/simpleAgreementCreationParameters.json @@ -0,0 +1 @@ +{"template_id":"iMindsServiceTemplate","user":"imauser"} diff --git a/sla/samples/templateCreate.bat b/sla/samples/templateCreate.bat new file mode 100755 index 00000000..56c2c96d --- /dev/null +++ b/sla/samples/templateCreate.bat @@ -0,0 +1 @@ +curl -u normal_user:password -H "Content-type: application/xml" -d@TemplateIMindsService.xml localhost:8080/sla-service/templateso -X POST \ No newline at end of file diff --git a/sla/sla_utils/bin/load-samples.sh b/sla/sla_utils/bin/load-samples.sh new file mode 100755 index 00000000..c316129f --- /dev/null +++ b/sla/sla_utils/bin/load-samples.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# +# To be executed from application root path +# + +#cmd=$($(grep SLA_MANAGER sladashboard/settings.py) && eval $cmd & print $SLA_MANAGER)) +#eval $(grep SLA_MANAGER_URL sladashboard/settings.py) +#echo SLA_MANAGER_URL=$SLA_MANAGER_URL + +SLA_MANAGER_URL="http://localhost:8080/sla-service" + +# +echo \#Add provider virtualwall +# +curl -H "Content-type: application/xml" -d @samples/provider-virtualwall.xml $SLA_MANAGER_URL/providers -X POST + +# +echo \#Add provider wiLab2 +# +curl -H "Content-type: application/xml" -d @samples/provider-wilab2.xml $SLA_MANAGER_URL/providers -X POST + +# +echo \#Add template +# +curl -H "Content-type: application/xml" -d @samples/template.xml $SLA_MANAGER_URL/templates -X POST + +# +echo \#Add agreement03 +# +curl -H "Content-type: application/xml" -d @samples/agreement03.xml $SLA_MANAGER_URL/agreements -X POST +curl -H "Content-type: application/xml" -d @samples/enforcement03.xml $SLA_MANAGER_URL/enforcements -X POST +#curl $SLA_MANAGER_URL/enforcements/agreement03/start -X PUT + +# +echo \#Add agreement04 +# +curl -H "Content-type: application/xml" -d @samples/agreement04.xml $SLA_MANAGER_URL/agreements -X POST +curl -H "Content-type: application/xml" -d @samples/enforcement04.xml $SLA_MANAGER_URL/enforcements -X POST +#curl $SLA_MANAGER_URL/enforcements/agreement04/start -X PUT + +# +#echo \#Add agreement05 +# +#curl -H "Content-type: application/xml" -d@samples/agreement05.xml $SLA_MANAGER_URL/agreements -X POST +#curl -d@samples/enforcement05.xml -H "Content-type: application/xml" $SLA_MANAGER_URL/enforcements -X POST +#curl $SLA_MANAGER_URL/enforcements/agreement05/start -X PUT + diff --git a/sla/sla_utils/bin/startagreement.sh b/sla/sla_utils/bin/startagreement.sh new file mode 100755 index 00000000..b9736fc9 --- /dev/null +++ b/sla/sla_utils/bin/startagreement.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +if [ $# -eq 1 ] ; then + curl localhost:8080/sla-service/enforcements/agreement0$1/start -X PUT +else + curl localhost:8080/sla-service/enforcements/agreement03/start -X PUT + curl localhost:8080/sla-service/enforcements/agreement04/start -X PUT +fi + diff --git a/sla/sla_utils/bin/stopagreement.sh b/sla/sla_utils/bin/stopagreement.sh new file mode 100755 index 00000000..c431f50d --- /dev/null +++ b/sla/sla_utils/bin/stopagreement.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +if [ $# -eq 1 ] ; then + curl localhost:8080/sla-service/enforcements/agreement0$1/stop -X PUT +else + curl localhost:8080/sla-service/enforcements/agreement03/stop -X PUT + curl localhost:8080/sla-service/enforcements/agreement04/stop -X PUT +fi + diff --git a/sla/sla_utils/samples/agreement03.xml b/sla/sla_utils/samples/agreement03.xml new file mode 100755 index 00000000..b6b7a9b0 --- /dev/null +++ b/sla/sla_utils/samples/agreement03.xml @@ -0,0 +1,55 @@ + + + + ExampleAgreement + + experimenter01 + virtualwall + AgreementResponder + 2014-03-07T12:00:00 + template + Testbed_guarantee_0.99_Uptime_rate_for_0.99_rate_of_the_resources_during_the_sliver + + + + + + + + qos:UpTime + + + qos:Performance + + + + + + + + + + UpTime + + {"constraint" : "UpTime GT 0.99"} + + + + + + + + + + + Performance + + {"constraint" : "Performance GT 0.99"} + + + + + + + diff --git a/sla/sla_utils/samples/agreement04.xml b/sla/sla_utils/samples/agreement04.xml new file mode 100755 index 00000000..fab041be --- /dev/null +++ b/sla/sla_utils/samples/agreement04.xml @@ -0,0 +1,55 @@ + + + + ExampleAgreement + + experimenter01 + wiLab2 + AgreementResponder + 2014-03-07T12:00:00 + template + Testbed_guarantee_0.99_Uptime_rate_for_0.99_rate_of_the_resources_during_the_sliver + + + + + + + + qos:UpTime + + + qos:Performance + + + + + + + + + + UpTime + + {"constraint" : "UpTime GT 0.99"} + + + + + + + + + + + Performance + + {"constraint" : "Performance GT 0.99"} + + + + + + + diff --git a/sla/sla_utils/samples/enforcement03.xml b/sla/sla_utils/samples/enforcement03.xml new file mode 100755 index 00000000..8f432672 --- /dev/null +++ b/sla/sla_utils/samples/enforcement03.xml @@ -0,0 +1,4 @@ + + agreement03 + false + diff --git a/sla/sla_utils/samples/enforcement04.xml b/sla/sla_utils/samples/enforcement04.xml new file mode 100755 index 00000000..bf2f5a96 --- /dev/null +++ b/sla/sla_utils/samples/enforcement04.xml @@ -0,0 +1,4 @@ + + agreement04 + false + diff --git a/sla/sla_utils/samples/old/agreement01.xml b/sla/sla_utils/samples/old/agreement01.xml new file mode 100755 index 00000000..547d149f --- /dev/null +++ b/sla/sla_utils/samples/old/agreement01.xml @@ -0,0 +1,44 @@ + + + ExampleAgreement + + RandomClient + Provider01 + + AgreementResponder + 2014-03-07T12:00 + contract-template-2007-12-04 + + + + + + + + + + + + + qos:ResponseTime + + + + + + + + + + ResponseTime + {"contraint" : "ResponseTime LT 100"} + + + + + + diff --git a/sla/sla_utils/samples/old/agreement02.xml b/sla/sla_utils/samples/old/agreement02.xml new file mode 100755 index 00000000..707bcdaf --- /dev/null +++ b/sla/sla_utils/samples/old/agreement02.xml @@ -0,0 +1,77 @@ + + + + ExampleAgreement + + RandomClient + provider-prueba + + AgreementResponder + 2014-03-07T12:00:00 + template02 + + + + + + DSL expression + + + DSL expression + + + + + + + + + qos:ResponseTime + + + qos:Performance + + + + + + + + + + ResponseTime + {"constraint" : "ResponseTime LT 0.9"} + + + + + + + + Performance + {"constraint" : "Performance GT 0.1"} + + + + 3 + + + 10 + + EUR + 99 + + + + + + + + + + diff --git a/sla/sla_utils/samples/old/agreement03_old.xml b/sla/sla_utils/samples/old/agreement03_old.xml new file mode 100755 index 00000000..291ff4c5 --- /dev/null +++ b/sla/sla_utils/samples/old/agreement03_old.xml @@ -0,0 +1,55 @@ + + + + ExampleAgreement + + experimenter01 + virtualwall + AgreementResponder + 2014-03-07T12:00:00 + template02 + Testbed_guarantee_0.75_Uptime_rate_for_0.8_rate_of_the_resources_during_the_sliver + + + + + + + + service-prueba/ResponseTime + + + service-prueba/Performance + + + + + + + + ResponseTime + + {"constraint" : "ResponseTime BETWEEN (0, 200)"} + + + + + + + + + Performance + + {"constraint" : "Performance BETWEEN (0.1,1)"} + + + + + + + diff --git a/sla/sla_utils/samples/old/agreement04_old.xml b/sla/sla_utils/samples/old/agreement04_old.xml new file mode 100755 index 00000000..28d3dd63 --- /dev/null +++ b/sla/sla_utils/samples/old/agreement04_old.xml @@ -0,0 +1,64 @@ + + + + ExampleAgreement + + experimenter01 + wiLab2 + AgreementResponder + 2014-03-07T12:00:00 + template02 + Testbed_guarantee_0.80_uptime_rate_for_0.75_rate_of_the_resources_during_the_sliver + + + + + + + metric1 + + + metric2 + + + metric3 + + + + + + + + metric1 + + {"constraint" : "metric1 BETWEEN (0.1, 1)"} + + + + + + + + + metric2 + + {"constraint" : "metric2 BETWEEN (0.15, 1)"} + + + + + + + + + metric3 + + {"constraint" : "metric3 BETWEEN (0.2, 1)"} + + + + + + + diff --git a/sla/sla_utils/samples/old/agreement05.xml b/sla/sla_utils/samples/old/agreement05.xml new file mode 100755 index 00000000..bd7c35c7 --- /dev/null +++ b/sla/sla_utils/samples/old/agreement05.xml @@ -0,0 +1,78 @@ + + + + ExampleAgreement + + client-prueba + f4c993580-03fe-41eb-8a21-a56709f9370f + AgreementResponder + 2014-03-07T12:00:00 + template02 + service5 + + + + + + + metric1 + + + metric2 + + + metric3 + + + metric4 + + + + + + + + metric1 + + {"constraint" : "metric1 BETWEEN (0.05, 1)"} + + + + + + + + + metric2 + + {"constraint" : "metric2 BETWEEN (0.1, 1)"} + + + + + + + + + metric3 + + {"constraint" : "metric3 BETWEEN (0.15, 1)"} + + + + + + + + + metric4 + + {"constraint" : "metric4 BETWEEN (0.2, 1)"} + + + + + + + diff --git a/sla/sla_utils/samples/old/agreement05_old.xml b/sla/sla_utils/samples/old/agreement05_old.xml new file mode 100755 index 00000000..777a1e60 --- /dev/null +++ b/sla/sla_utils/samples/old/agreement05_old.xml @@ -0,0 +1,78 @@ + + + + ExampleAgreement + + experimenter01 + virtualwall + AgreementResponder + 2014-03-07T12:00:00 + template02 + service5 + + + + + + + metric1 + + + metric2 + + + metric3 + + + metric4 + + + + + + + + metric1 + + {"constraint" : "metric1 BETWEEN (0.05, 1)"} + + + + + + + + + metric2 + + {"constraint" : "metric2 BETWEEN (0.1, 1)"} + + + + + + + + + metric3 + + {"constraint" : "metric3 BETWEEN (0.15, 1)"} + + + + + + + + + metric4 + + {"constraint" : "metric4 BETWEEN (0.2, 1)"} + + + + + + + diff --git a/sla/sla_utils/samples/old/enforcement.xml b/sla/sla_utils/samples/old/enforcement.xml new file mode 100755 index 00000000..bf2f5a96 --- /dev/null +++ b/sla/sla_utils/samples/old/enforcement.xml @@ -0,0 +1,4 @@ + + agreement04 + false + diff --git a/sla/sla_utils/samples/old/enforcement02.xml b/sla/sla_utils/samples/old/enforcement02.xml new file mode 100755 index 00000000..e81c83d4 --- /dev/null +++ b/sla/sla_utils/samples/old/enforcement02.xml @@ -0,0 +1,4 @@ + + agreement02 + true + diff --git a/sla/sla_utils/samples/old/enforcement05.xml b/sla/sla_utils/samples/old/enforcement05.xml new file mode 100755 index 00000000..1b4dc7e6 --- /dev/null +++ b/sla/sla_utils/samples/old/enforcement05.xml @@ -0,0 +1,4 @@ + + agreement05 + true + diff --git a/sla/sla_utils/samples/old/template01.xml b/sla/sla_utils/samples/old/template01.xml new file mode 100755 index 00000000..90b24c14 --- /dev/null +++ b/sla/sla_utils/samples/old/template01.xml @@ -0,0 +1,78 @@ + + + + ExampleTemplate + + Provider + AgreementInitiator + 2013-12-15-1200 + contract-template-2013-12-15 + + + + + + A GPS service + + + operation to call to get the coords + + + + + http://www.gps.com/coordsservice/getcoords + gps:CoordsRequest + + + + + + + qos:ResponseTime + + + + + + + qos:CoordDerivation + + + + + + + http://www.gps.com/coordsservice/getcoords + + + applied when current time in week working hours + + + + FastResponseTime + + //Variable/@Name="ResponseTime" LOWERTHAN 1 second + + + + + + + diff --git a/sla/sla_utils/samples/provider-virtualwall.xml b/sla/sla_utils/samples/provider-virtualwall.xml new file mode 100755 index 00000000..6299a9f7 --- /dev/null +++ b/sla/sla_utils/samples/provider-virtualwall.xml @@ -0,0 +1,5 @@ + + + virtualwall + virtualwall + diff --git a/sla/sla_utils/samples/provider-wilab2.xml b/sla/sla_utils/samples/provider-wilab2.xml new file mode 100755 index 00000000..26716563 --- /dev/null +++ b/sla/sla_utils/samples/provider-wilab2.xml @@ -0,0 +1,5 @@ + + + wiLab2 + wiLab2 + diff --git a/sla/sla_utils/samples/template.xml b/sla/sla_utils/samples/template.xml new file mode 100755 index 00000000..ab3ce59b --- /dev/null +++ b/sla/sla_utils/samples/template.xml @@ -0,0 +1,69 @@ + + + ExampleTemplate2 + + 2014-03-07T12:00:00:000 + + + + + + DSL expression + + + DSL expression + + + + + + + + + qos:ResponseTime + + + qos:Performance + + + + + + + + + + ResponseTime + {"constraint" : "ResponseTime LT qos:ResponseTime"} + + + + + + + + Performance + {"constraint" : "Performance GT qos:Performance"} + + + + 3 + + + 10 + + EUR + 99 + + + + + + + + + + diff --git a/sla/slaclient/__init__.py b/sla/slaclient/__init__.py new file mode 100755 index 00000000..449f7cbd --- /dev/null +++ b/sla/slaclient/__init__.py @@ -0,0 +1 @@ +__author__ = 'a145034' diff --git a/sla/slaclient/restclient.py b/sla/slaclient/restclient.py new file mode 100755 index 00000000..cdada8a5 --- /dev/null +++ b/sla/slaclient/restclient.py @@ -0,0 +1,458 @@ +# -*- coding: utf-8 -*- + +import requests + +from requests.auth import HTTPBasicAuth + +import xmlconverter +import wsag_model + +from django.conf import settings + + +"""REST client to SLA Manager. + +Contains a generic rest client and wrappers over this generic client +for each resource. + +Each resource client implements business-like() functions, but +returns a tuple (output, requests.Response) + +The resource clients are initialized with the rooturl and a path, which +are combined to build the resource url. The path is defaulted to the known +resource path. So, for example, to create a agreements client: + +c = Agreements("http://localhost/slagui-service") + +A Factory facility is provided to create resource client instances. The +Factory uses "rooturl" module variable to use as rooturl parameter. + +restclient.rooturl = "http://localhost/slagui-service" +c = restclient.Factory.agreements() + +""" + +_PROVIDERS_PATH = "providerso" +_AGREEMENTS_PATH = "agreementso" +_TEMPLATES_PATH = "templateso" +_VIOLATIONS_PATH = "violationso" +_ENFORCEMENTJOBS_PATH = "enforcements" + +rooturl = settings.SLA_MANAGER_URL + +# SLA_MANAGER_USER = "normal_user" +# SLA_MANAGER_PASSWORD = "password" + +class Factory(object): + @staticmethod + def agreements(): + """Returns aREST client for Agreements + + :rtype : Agreements + """ + return Agreements(rooturl) + + @staticmethod + def providers(): + """Returns aREST client for Providers + + :rtype : Providers + """ + return Providers(rooturl) + + @staticmethod + def violations(): + """Returns aREST client for Violations + + :rtype : Violations + """ + return Violations(rooturl) + + @staticmethod + def templates(): + """Returns aREST client for Violations + + :rtype : Violations + """ + return Templates(rooturl) + + @staticmethod + def enforcements(): + """Returns aREST client for Enforcements jobs + + :rtype : Enforcements + """ + return Enforcements(rooturl) + +class Client(object): + + def __init__(self, root_url): + + """Generic rest client using requests library + + Each operation mimics the corresponding "requests" operation (arguments + and return) + + :param str root_url: this url is used as prefix in all subsequent + requests + """ + self.rooturl = root_url + + def get(self, path, **kwargs): + """Just a wrapper over request.get, just in case. + + Returns a requests.Response + + :rtype : request.Response + :param str path: remaining path from root url; + empty if desired path equal to rooturl. + :param kwargs: arguments to requests.get + + Example: + c = Client("http://localhost:8080/service") + c.get("/resource", headers = { "accept": "application/json" }) + """ + url = _buildpath_(self.rooturl, path) + kwargs["auth"] = HTTPBasicAuth(settings.SLA_MANAGER_USER, settings.SLA_MANAGER_PASSWORD) + result = requests.get(url, **kwargs) + print "GET {} {} {}".format( + result.url, result.status_code, result.text[0:70]) + return result + + def post(self, path, data=None, **kwargs): + """Just a wrapper over request.post, just in case + + :rtype : request.Response + :param str path: remaining path from root url; + empty if desired path equal to rooturl. + :param dict[str, str] kwargs: arguments to requests.post + + Example: + c = Client("http://localhost:8080/service") + c.post( + '/resource', + '{ "id": "1", "name": "provider-a" }', + headers = { + "content-type": "application/json", + "accept": "application/xml" + } + ) + """ + url = _buildpath_(self.rooturl, path) + kwargs["auth"] = HTTPBasicAuth(settings.SLA_MANAGER_USER, settings.SLA_MANAGER_PASSWORD) + result = requests.post(url, data, **kwargs) + location = result.headers["location"] \ + if "location" in result.headers else "" + print "POST {} {} Location: {}".format( + result.url, result.status_code, location) + return result + + + +class _Resource(object): + + def __init__(self, url, converter): + """Provides some common operations over resources. + + The operations return a structured representation of the resource. + + :param str url: url to the resource + :param Converter converter: resouce xml converter + + Some attributes are initialized to be used from the owner if needed: + * client: Client instance + * converter: resource xml converter + * listconverter: list of resources xml converter + """ + self.client = Client(url) + self.converter = converter + self.listconverter = xmlconverter.ListConverter(self.converter) + + @staticmethod + def _processresult(r, converter): + + """Generic processing of the REST call. + + If no errors, tries to convert the result to a destination entity. + + :param r requests: + :param converter Converter: + """ + if r.status_code == 404: + return None + + content_type = r.headers.get('content-type', '') + + print("content-type = " + content_type) + if content_type == 'application/json': + result = r.json() + elif content_type == 'application/xml': + xml = r.text + result = xmlconverter.convertstring(converter, xml) + else: + result = r.text + return result + + def getall(self): + """Get all resources + + """ + r = self.client.get("") + resources = self._processresult(r, self.listconverter) + return resources, r + + def getbyid(self, id): + """Get resource 'id'""" + r = self.client.get(id) + resource = _Resource._processresult(r, self.converter) + return resource, r + + def get(self, params): + """Generic query over resource: GET /resource?q1=v1&q2=v2... + + :param dict[str,str] params: values to pass as get parameters + """ + r = self.client.get("", params=params) + resources = self._processresult(r, self.listconverter) + return resources, r + + def create(self, body, **kwargs): + """Creates (POST method) a resource. + + It should be convenient to set content-type header. + + Usage: + resource.create(body, headers={'content-type': 'application/xml'}) + """ + r = self.client.post("", body, **kwargs) + r.raise_for_status() + return r + + +class Agreements(object): + + def __init__(self, root_url, path=_AGREEMENTS_PATH): + """Business methods for Agreement resource + :param str root_url: url to the root of resources + :param str path: path to resource from root_url + + The final url to the resource is root_url + "/" + path + """ + resourceurl = _buildpath_(root_url, path) + converter = xmlconverter.AgreementConverter() + self.res = _Resource(resourceurl, converter) + + def getall(self): + """ + Get all agreements + + :rtype : list[wsag_model.Agreement] + """ + return self.res.getall() + + def getbyid(self, agreementid): + """Get an agreement + + :rtype : wsag_model.Agreement + """ + return self.res.getbyid(agreementid) + + def getbyconsumer(self, consumerid): + """Get a consumer's agreements + + :rtype : list[wsag_model.Agreement] + """ + return self.res.get(dict(consumerId=consumerid)) + + def getbyprovider(self, providerid): + """Get the agreements served by a provider + + :rtype : list[wsag_model.Agreement] + """ + return self.res.get(dict(providerId=providerid)) + + def getstatus(self, agreementid): + """Get guarantee status of an agreement + + :param str agreementid : + :rtype : wsag_model.AgreementStatus + """ + path = _buildpath_(agreementid, "guaranteestatus") + r = self.res.client.get(path, headers={'accept': 'application/json'}) + json_obj = r.json() + status = wsag_model.AgreementStatus.json_decode(json_obj) + + return status, r + + def create(self, agreement): + """Create a new agreement + + :param str agreement: sla template in ws-agreement format. + """ + return self.res.create(agreement) + +class Templates(object): + + def __init__(self, root_url, path=_TEMPLATES_PATH): + """Business methods for Templates resource + :param str root_url: url to the root of resources + :param str path: path to resource from root_url + + The final url to the resource is root_url + "/" + path + """ + resourceurl = _buildpath_(root_url, path) + converter = xmlconverter.AgreementConverter() + self.res = _Resource(resourceurl, converter) + + def getall(self): + """ Get all templates + + :rtype : list[wsag_model.Template] + """ + return self.res.getall() + + def getbyid(self, provider_id): + """Get a template + + :rtype: wsag_model.Template + """ + return self.res.getbyid(provider_id) + + def create(self, template): + """Create a new template + + :param str template: sla template in ws-agreement format. + """ + self.res.create(template) + +class Providers(object): + + def __init__(self, root_url, path=_PROVIDERS_PATH): + """Business methods for Providers resource + :param str root_url: url to the root of resources + :param str path: path to resource from root_url + + The final url to the resource is root_url + "/" + path + """ + resourceurl = _buildpath_(root_url, path) + converter = xmlconverter.ProviderConverter() + self.res = _Resource(resourceurl, converter) + + def getall(self): + """ Get all providers + + :rtype : list[wsag_model.Provider] + """ + return self.res.getall() + + def getbyid(self, provider_id): + """Get a provider + + :rtype: wsag_model.Provider + """ + return self.res.getbyid(provider_id) + + def create(self, provider): + """Create a new provider + + :type provider: wsag_model.Provider + """ + body = provider.to_xml() + return self.res.create(body) + +class Violations(object): + + def __init__(self, root_url, path=_VIOLATIONS_PATH): + """Business methods for Violation resource + :param str root_url: url to the root of resources + :param str path: path to resource from root_url + + The final url to the resource is root_url + "/" + path + """ + resourceurl = _buildpath_(root_url, path) + converter = xmlconverter.ViolationConverter() + self.res = _Resource(resourceurl, converter) + + def getall(self): + """ Get all violations + :rtype : list[wsag_model.Violation] + """ + return self.res.getall() + + def getbyid(self, violationid): + """Get a violation + + :rtype : wsag_model.Violation + """ + return self.res.getbyid(violationid) + + def getbyagreement(self, agreement_id, term=None): + """Get the violations of an agreement. + + :param str agreement_id: + :param str term: optional GuaranteeTerm name. If not specified, + violations from all terms will be returned + :rtype: list[wsag_model.Violation] + """ + return self.res.get( + {"agreementId": agreement_id, "guaranteeTerm": term}) + + +class Enforcements(object): + + def __init__(self, root_url, path=_ENFORCEMENTJOBS_PATH): + """Business methods for Violation resource + :param str root_url: url to the root of resources + :param str path: path to resource from root_url + + The final url to the resource is root_url + "/" + path + """ + resourceurl = _buildpath_(root_url, path) + converter = xmlconverter.EnforcementConverter() + self.res = _Resource(resourceurl, converter) + + def getall(self): + """ Get all Enforcements + :rtype : list[wsag_model.Violation] + """ + return self.res.getall() + + def getbyagreement(self, agreement_id): + """Get the enforcement of an agreement. + + :param str agreement_id: + + :rtype: list[wsag_model.Enforcement] + """ + return self.res.getbyid(agreement_id) + + +def _buildpath_(*paths): + return "/".join(paths) + + +def main(): + # + # Move to test + # + global rooturl + rooturl = "http://127.0.0.1:8080/sla-service" + + + c = Factory.templates() + #r = c.getall() + #r = c.getbyid("noexiste") + #r = c.getstatus("agreement03") + #print r + + #r = c.getbyconsumer('RandomClient') + r = c.getbyid("template02") + + + print r + + +if __name__ == "__main__": + main() + + diff --git a/sla/slaclient/restclient_nosecurity.py b/sla/slaclient/restclient_nosecurity.py new file mode 100755 index 00000000..e373ccce --- /dev/null +++ b/sla/slaclient/restclient_nosecurity.py @@ -0,0 +1,318 @@ +# -*- coding: utf-8 -*- + +import requests + +import xmlconverter +import wsag_model + + +"""REST client to SLA Manager. + +Contains a generic rest client and wrappers over this generic client +for each resource. + +Each resource client implements business-like() functions, but +returns a tuple (output, requests.Response) + +The resource clients are initialized with the rooturl and a path, which +are combined to build the resource url. The path is defaulted to the known +resource path. So, for example, to create a agreements client: + +c = Agreements("http://localhost/slagui-service") + +A Factory facility is provided to create resource client instances. The +Factory uses "rooturl" module variable to use as rooturl parameter. + +restclient.rooturl = "http://localhost/slagui-service" +c = restclient.Factory.agreements() + +""" + +_PROVIDERS_PATH = "providers" +_AGREEMENTS_PATH = "agreements" +_VIOLATIONS_PATH = "violations" +_ENFORCEMENTJOBS_PATH = "enforcementjobs" + +rooturl = "" + + +class Factory(object): + @staticmethod + def agreements(): + """Returns aREST client for Agreements + + :rtype : Agreements + """ + return Agreements(rooturl) + + @staticmethod + def providers(): + """Returns aREST client for Providers + + :rtype : Providers + """ + return Providers(rooturl) + + @staticmethod + def violations(): + """Returns aREST client for Violations + + :rtype : Violations + """ + return Violations(rooturl) + +class Client(object): + + def __init__(self, root_url): + + """Generic rest client using requests library + + Each operation mimics the corresponding "requests" operation (arguments + and return) + + :param str root_url: this url is used as prefix in all subsequent + requests + """ + self.rooturl = root_url + + def get(self, path, **kwargs): + """Just a wrapper over request.get, just in case. + + Returns a requests.Response + + :rtype : request.Response + :param str path: remaining path from root url; + empty if desired path equal to rooturl. + :param kwargs: arguments to requests.get + + Example: + c = Client("http://localhost:8080/service") + c.get("/resource", headers = { "accept": "application/json" }) + """ + url = _buildpath_(self.rooturl, path) + result = requests.get(url, **kwargs) + print "GET {} {} {}".format( + result.url, result.status_code, result.text[0:70]) + return result + +class _Resource(object): + + def __init__(self, url, converter): + """Provides some common operations over resources. + + The operations return a structured representation of the resource. + + :param str url: url to the resource + :param Converter converter: resouce xml converter + + Some attributes are initialized to be used from the owner if needed: + * client: Client instance + * converter: resource xml converter + * listconverter: list of resources xml converter + """ + self.client = Client(url) + self.converter = converter + self.listconverter = xmlconverter.ListConverter(self.converter) + + @staticmethod + def _processresult(r, converter): + + """Generic processing of the REST call. + + If no errors, tries to convert the result to a destination entity. + + :param r requests: + :param converter Converter: + """ + if r.status_code == 404: + return None + + content_type = r.headers.get('content-type', '') + + print("content-type = " + content_type) + if content_type == 'application/json': + result = r.json() + elif content_type == 'application/xml': + xml = r.text + result = xmlconverter.convertstring(converter, xml) + else: + result = r.text + return result + + def getall(self): + """Get all resources + + """ + r = self.client.get("") + resources = self._processresult(r, self.listconverter) + return resources, r + + def getbyid(self, id): + """Get resource 'id'""" + r = self.client.get(id) + resource = _Resource._processresult(r, self.converter) + return resource, r + + def get(self, params): + """Generic query over resource: GET /resource?q1=v1&q2=v2... + + :param dict[str,str] params: values to pass as get parameters + """ + r = self.client.get("", params=params) + resources = self._processresult(r, self.listconverter) + return resources, r + +class Agreements(object): + + def __init__(self, root_url, path=_AGREEMENTS_PATH): + """Business methods for Agreement resource + :param str root_url: url to the root of resources + :param str path: path to resource from root_url + + The final url to the resource is root_url + "/" + path + """ + resourceurl = _buildpath_(root_url, path) + converter = xmlconverter.AgreementConverter() + self.res = _Resource(resourceurl, converter) + + def getall(self): + """ + Get all agreements + + :rtype : list[wsag_model.Agreement] + """ + return self.res.getall() + + def getbyid(self, agreementid): + """Get an agreement + + :rtype : wsag_model.Agreement + """ + return self.res.getbyid(agreementid) + + def getbyconsumer(self, consumerid): + """Get a consumer's agreements + + :rtype : list[wsag_model.Agreement] + """ + return self.res.get(dict(consumerId=consumerid)) + + def getbyprovider(self, providerid): + """Get the agreements served by a provider + + :rtype : list[wsag_model.Agreement] + """ + return self.res.get(dict(providerId=providerid)) + + def getstatus(self, agreementid): + """Get guarantee status of an agreement + + :param str agreementid : + :rtype : wsag_model.AgreementStatus + """ + path = _buildpath_(agreementid, "guaranteestatus") + r = self.res.client.get(path, headers={'accept': 'application/json'}) + json_obj = r.json() + status = wsag_model.AgreementStatus.json_decode(json_obj) + + return status, r + +class Providers(object): + + def __init__(self, root_url, path=_PROVIDERS_PATH): + """Business methods for Providers resource + :param str root_url: url to the root of resources + :param str path: path to resource from root_url + + The final url to the resource is root_url + "/" + path + """ + resourceurl = _buildpath_(root_url, path) + converter = xmlconverter.ProviderConverter() + self.res = _Resource(resourceurl, converter) + + def getall(self): + """ Get all providers + + :rtype : list[wsag_model.Provider] + """ + return self.res.getall() + + def getbyid(self, provider_id): + """Get a provider + + :rtype: wsag_model.Provider + """ + return self.res.getbyid(provider_id) + +class Violations(object): + + def __init__(self, root_url, path=_VIOLATIONS_PATH): + """Business methods for Violation resource + :param str root_url: url to the root of resources + :param str path: path to resource from root_url + + The final url to the resource is root_url + "/" + path + """ + resourceurl = _buildpath_(root_url, path) + converter = xmlconverter.ViolationConverter() + self.res = _Resource(resourceurl, converter) + + def getall(self): + """ Get all violations + :rtype : list[wsag_model.Violation] + """ + return self.res.getall() + + def getbyid(self, violationid): + """Get a violation + + :rtype : wsag_model.Violation + """ + return self.res.getbyid(violationid) + + def getbyagreement(self, agreement_id, term=None): + """Get the violations of an agreement. + + :param str agreement_id: + :param str term: optional GuaranteeTerm name. If not specified, + violations from all terms will be returned + :rtype: list[wsag_model.Violation] + """ + return self.res.get( + {"agreementId": agreement_id, "guaranteeTerm": term}) + + +def _buildpath_(*paths): + return "/".join(paths) + + +def main(): + # + # Move to test + # + global rooturl + rooturl = "http://10.0.2.2:8080/sla-service" + + c = Factory.agreements() + #r = c.getall() + #r = c.getbyid("noexiste") + #r = c.getstatus("agreement03") + #print r + + #r = c.getbyconsumer('RandomClient') + + c = Providers(rooturl) + r = c.getall() + + c = Violations(rooturl) + #r = c.getall() + r_ = c.getbyagreement("agreement03", "GT_Otro") + r_ = c.getbyid('cf41011d-9f30-4ebc-a967-30b4ea928192') + + print r_ + + +if __name__ == "__main__": + main() + + diff --git a/sla/slaclient/restclient_old.py b/sla/slaclient/restclient_old.py new file mode 100755 index 00000000..e373ccce --- /dev/null +++ b/sla/slaclient/restclient_old.py @@ -0,0 +1,318 @@ +# -*- coding: utf-8 -*- + +import requests + +import xmlconverter +import wsag_model + + +"""REST client to SLA Manager. + +Contains a generic rest client and wrappers over this generic client +for each resource. + +Each resource client implements business-like() functions, but +returns a tuple (output, requests.Response) + +The resource clients are initialized with the rooturl and a path, which +are combined to build the resource url. The path is defaulted to the known +resource path. So, for example, to create a agreements client: + +c = Agreements("http://localhost/slagui-service") + +A Factory facility is provided to create resource client instances. The +Factory uses "rooturl" module variable to use as rooturl parameter. + +restclient.rooturl = "http://localhost/slagui-service" +c = restclient.Factory.agreements() + +""" + +_PROVIDERS_PATH = "providers" +_AGREEMENTS_PATH = "agreements" +_VIOLATIONS_PATH = "violations" +_ENFORCEMENTJOBS_PATH = "enforcementjobs" + +rooturl = "" + + +class Factory(object): + @staticmethod + def agreements(): + """Returns aREST client for Agreements + + :rtype : Agreements + """ + return Agreements(rooturl) + + @staticmethod + def providers(): + """Returns aREST client for Providers + + :rtype : Providers + """ + return Providers(rooturl) + + @staticmethod + def violations(): + """Returns aREST client for Violations + + :rtype : Violations + """ + return Violations(rooturl) + +class Client(object): + + def __init__(self, root_url): + + """Generic rest client using requests library + + Each operation mimics the corresponding "requests" operation (arguments + and return) + + :param str root_url: this url is used as prefix in all subsequent + requests + """ + self.rooturl = root_url + + def get(self, path, **kwargs): + """Just a wrapper over request.get, just in case. + + Returns a requests.Response + + :rtype : request.Response + :param str path: remaining path from root url; + empty if desired path equal to rooturl. + :param kwargs: arguments to requests.get + + Example: + c = Client("http://localhost:8080/service") + c.get("/resource", headers = { "accept": "application/json" }) + """ + url = _buildpath_(self.rooturl, path) + result = requests.get(url, **kwargs) + print "GET {} {} {}".format( + result.url, result.status_code, result.text[0:70]) + return result + +class _Resource(object): + + def __init__(self, url, converter): + """Provides some common operations over resources. + + The operations return a structured representation of the resource. + + :param str url: url to the resource + :param Converter converter: resouce xml converter + + Some attributes are initialized to be used from the owner if needed: + * client: Client instance + * converter: resource xml converter + * listconverter: list of resources xml converter + """ + self.client = Client(url) + self.converter = converter + self.listconverter = xmlconverter.ListConverter(self.converter) + + @staticmethod + def _processresult(r, converter): + + """Generic processing of the REST call. + + If no errors, tries to convert the result to a destination entity. + + :param r requests: + :param converter Converter: + """ + if r.status_code == 404: + return None + + content_type = r.headers.get('content-type', '') + + print("content-type = " + content_type) + if content_type == 'application/json': + result = r.json() + elif content_type == 'application/xml': + xml = r.text + result = xmlconverter.convertstring(converter, xml) + else: + result = r.text + return result + + def getall(self): + """Get all resources + + """ + r = self.client.get("") + resources = self._processresult(r, self.listconverter) + return resources, r + + def getbyid(self, id): + """Get resource 'id'""" + r = self.client.get(id) + resource = _Resource._processresult(r, self.converter) + return resource, r + + def get(self, params): + """Generic query over resource: GET /resource?q1=v1&q2=v2... + + :param dict[str,str] params: values to pass as get parameters + """ + r = self.client.get("", params=params) + resources = self._processresult(r, self.listconverter) + return resources, r + +class Agreements(object): + + def __init__(self, root_url, path=_AGREEMENTS_PATH): + """Business methods for Agreement resource + :param str root_url: url to the root of resources + :param str path: path to resource from root_url + + The final url to the resource is root_url + "/" + path + """ + resourceurl = _buildpath_(root_url, path) + converter = xmlconverter.AgreementConverter() + self.res = _Resource(resourceurl, converter) + + def getall(self): + """ + Get all agreements + + :rtype : list[wsag_model.Agreement] + """ + return self.res.getall() + + def getbyid(self, agreementid): + """Get an agreement + + :rtype : wsag_model.Agreement + """ + return self.res.getbyid(agreementid) + + def getbyconsumer(self, consumerid): + """Get a consumer's agreements + + :rtype : list[wsag_model.Agreement] + """ + return self.res.get(dict(consumerId=consumerid)) + + def getbyprovider(self, providerid): + """Get the agreements served by a provider + + :rtype : list[wsag_model.Agreement] + """ + return self.res.get(dict(providerId=providerid)) + + def getstatus(self, agreementid): + """Get guarantee status of an agreement + + :param str agreementid : + :rtype : wsag_model.AgreementStatus + """ + path = _buildpath_(agreementid, "guaranteestatus") + r = self.res.client.get(path, headers={'accept': 'application/json'}) + json_obj = r.json() + status = wsag_model.AgreementStatus.json_decode(json_obj) + + return status, r + +class Providers(object): + + def __init__(self, root_url, path=_PROVIDERS_PATH): + """Business methods for Providers resource + :param str root_url: url to the root of resources + :param str path: path to resource from root_url + + The final url to the resource is root_url + "/" + path + """ + resourceurl = _buildpath_(root_url, path) + converter = xmlconverter.ProviderConverter() + self.res = _Resource(resourceurl, converter) + + def getall(self): + """ Get all providers + + :rtype : list[wsag_model.Provider] + """ + return self.res.getall() + + def getbyid(self, provider_id): + """Get a provider + + :rtype: wsag_model.Provider + """ + return self.res.getbyid(provider_id) + +class Violations(object): + + def __init__(self, root_url, path=_VIOLATIONS_PATH): + """Business methods for Violation resource + :param str root_url: url to the root of resources + :param str path: path to resource from root_url + + The final url to the resource is root_url + "/" + path + """ + resourceurl = _buildpath_(root_url, path) + converter = xmlconverter.ViolationConverter() + self.res = _Resource(resourceurl, converter) + + def getall(self): + """ Get all violations + :rtype : list[wsag_model.Violation] + """ + return self.res.getall() + + def getbyid(self, violationid): + """Get a violation + + :rtype : wsag_model.Violation + """ + return self.res.getbyid(violationid) + + def getbyagreement(self, agreement_id, term=None): + """Get the violations of an agreement. + + :param str agreement_id: + :param str term: optional GuaranteeTerm name. If not specified, + violations from all terms will be returned + :rtype: list[wsag_model.Violation] + """ + return self.res.get( + {"agreementId": agreement_id, "guaranteeTerm": term}) + + +def _buildpath_(*paths): + return "/".join(paths) + + +def main(): + # + # Move to test + # + global rooturl + rooturl = "http://10.0.2.2:8080/sla-service" + + c = Factory.agreements() + #r = c.getall() + #r = c.getbyid("noexiste") + #r = c.getstatus("agreement03") + #print r + + #r = c.getbyconsumer('RandomClient') + + c = Providers(rooturl) + r = c.getall() + + c = Violations(rooturl) + #r = c.getall() + r_ = c.getbyagreement("agreement03", "GT_Otro") + r_ = c.getbyid('cf41011d-9f30-4ebc-a967-30b4ea928192') + + print r_ + + +if __name__ == "__main__": + main() + + diff --git a/sla/slaclient/service/__init__.py b/sla/slaclient/service/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/sla/slaclient/service/fed4fire/__init__.py b/sla/slaclient/service/fed4fire/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/sla/slaclient/service/fed4fire/fed4fireservice.py b/sla/slaclient/service/fed4fire/fed4fireservice.py new file mode 100755 index 00000000..caecf26e --- /dev/null +++ b/sla/slaclient/service/fed4fire/fed4fireservice.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +"""Builds templates/agreements based on input data (in json format), submitting +to sla manager. + +It is intended as backend service for a rest interface. + +The json input must work together with the templates to form a valid template + or agreement for Xifi (be careful!) + +This (very simple) service is coupled to the way xifi is interpreting +ws-agreement. + + +""" +import json +import jsonparser +from sla.slaclient import wsag_model +from sla.slaclient import restclient +from sla.slaclient.templates.fed4fire.django.factory import Factory as TemplateFactory +import sla.slaclient.templates.fed4fire as fed4fire +from time import localtime, strftime +import uuid +class ServiceContext(object): + def __init__(self, restfactory = None, templatefactory=None): + """ + :type restfactory: restclient.Factory + """ + self.restfactory = restfactory + self.templatefactory = templatefactory + + +def createprovider(json_data, context): + """Creates a provider in the SlaManager. + :type json_data:str + :type context: ServiceContext + + An example input is: + { + "uuid": "f4c993580-03fe-41eb-8a21-a56709f9370f", + "name": "provider" + } + """ + json_obj = json.loads(json_data) + p = wsag_model.Provider.from_dict(json_obj) + provider_client = context.restfactory.providers() + provider_client.create(p) + + +def createtemplate(json_data, context): + """Creates a template in the SlaManager + + An example input is: + { + "template_id" : "template-id", + "template_name" : "template-name", + "provider" : "provider-1", + "service_id" : "service-id", + "expiration_time" : "2014-03-28T13:55:00Z", + "service_properties" : [ + { + "name" : "uptime", + "servicename" : "service-a", + "metric" : "xs:double", + "location" : "//service-a/uptime" + } + ] + } + + :type json_data:str + :type context: ServiceContext + """ + data = jsonparser.templateinput_from_json(json_data) + slatemplate = sla.slaclient.templates.fed4fire.render_slatemplate(data) + client = context.restfactory.templates() + client.create(slatemplate) + + +def createagreement(json_data, context): + """Creates an agreement in the SlaManager. + + The template with template_id is retrieved and the properties and some + context info is copied to the agreement. + + An example input is: + { + "template_id" : "template-id", + "agreement_id" : "agreement-id", + "expiration_time" : "2014-03-28T13:55:00Z", + "consumer" : "consumer-a", + "guarantees" : [ + { + "name" : "uptime", + "bounds" : [ "0", "1" ] + } + ] + } + :type json_data:str + :type context: ServiceContext + """ + client_templates = context.restfactory.templates() + + # Builds AgreementInput from json + data = jsonparser.agreementinput_from_json(json_data) + # Read template from manager + slatemplate, request = client_templates.getbyid(data.template_id) + # Copy (overriding if necessary) from template to AgreementInput + final_data = data.from_template(slatemplate) + slaagreement = fed4fire.render_slaagreement(final_data) + + client_agreements = context.restfactory.agreements() + return client_agreements.create(slaagreement) + + +def createagreementsimplified(template_id, user, expiration_time): + context = ServiceContext( + restclient.Factory(), + TemplateFactory() + ) + + agreement = { + "agreement_id": str(uuid.uuid4()), + "template_id": template_id, + "expiration_time": expiration_time, + "consumer": user, + } + + json_data = json.dumps(agreement) + + return createagreement(json_data, context) + +def main(): + createagreementsimplified("iMindsServiceWiLab2", "virtualwall", "2014-04-34T23:12:12") + + +if __name__ == "__main__": + main() + + diff --git a/sla/slaclient/service/fed4fire/jsonparser.py b/sla/slaclient/service/fed4fire/jsonparser.py new file mode 100755 index 00000000..3dc8fca3 --- /dev/null +++ b/sla/slaclient/service/fed4fire/jsonparser.py @@ -0,0 +1,120 @@ +""" + +""" +from sla.slaclient import wsag_model +import json +import dateutil.parser +from sla.slaclient.templates.fed4fire.fed4fire import AgreementInput +from sla.slaclient.templates.fed4fire.fed4fire import TemplateInput + + +def templateinput_from_json(json_data): + """Creates a TemplateInput from json data. + + :rtype: TemplateInput + + An example input is: + { + "agreement_id" : "agreement-id" + "agreement_name" : "agreement-name", + "template_id" : "template-id", + "provider" : "provider", + "service_id" : "service-id", + "expiration_time" : "2014-03-28T13:55:00Z", + "service_properties" : [ + { + "name" : "uptime", + "servicename" : "service-a", + "metric" : "xs:double", + "location" : "//service-a/uptime" + } + ] + } + """ + d = json.loads(json_data) + if "expiration_time" in d: + d["expiration_time"] = dateutil.parser.parse(d["expiration_time"]) + + t = TemplateInput( + template_id=d.get("template_id", None), + template_name=d.get("template_name", None), + provider=d.get("provider", None), + service_id=d.get("service_id"), + expiration_time=d.get("expiration_time", None), + service_properties=_json_parse_service_properties(d) + ) + return t + + +def agreementinput_from_json(json_data): + """Creates an AgreementInput from json data. + + :rtype: AgreementInput + + An example input is: + { + "agreement_id" : "agreement-id" + "agreement_name" : "agreement-name", + "template_id" : "template-id", + "consumer" : "consumer", + "provider" : "provider", + "service_id" : "service-id", + "expiration_time" : "2014-03-28T13:55:00Z", + "guarantees": [ + { + "name" : "uptime", + "bounds" : [ "0", "1" ] + } + ] + } + """ + d = json.loads(json_data) + if "expiration_time" in d: + d["expiration_time"] = dateutil.parser.parse(d["expiration_time"]) + + t = AgreementInput( + agreement_id=d.get("agreement_id", None), + agreement_name=d.get("agreement_name", None), + template_id=d.get("template_id", None), + consumer=d.get("consumer", None), + provider=d.get("provider", None), + service_id=d.get("service_id"), + expiration_time=d.get("expiration_time", None), + service_properties=_json_parse_service_properties(d), + guarantee_terms=_json_parse_guarantee_terms(d) + ) + return t + + +def _json_parse_service_properties(d): + """Parse service properties in a json and translates to Property. + :type d: dict(str, str) + :rtype: list(wsag_model.Agreement.Property) + """ + result = [] + for sp in d.get("service_properties", None) or (): + result.append( + wsag_model.Agreement.Property( + servicename=sp.get("servicename", None), + name=sp.get("name", None), + metric=sp.get("metric", None), + location=sp.get("location", None) + ) + ) + return result + + +def _json_parse_guarantee_terms(d): + """Parse guarantee terms in a son and translates to GuaranteeTerm. + :type d: dict(str, str) + :rtype: list(wsag_model.AgreementInput.GuaranteeTerm) + """ + result = [] + for term in d.get("guarantees", None) or (): + result.append( + AgreementInput.GuaranteeTerm( + metric_name=term["name"], + bounds=tuple(term["bounds"]) + ) + ) + return result \ No newline at end of file diff --git a/sla/slaclient/service/fed4fire/tests/__init__.py b/sla/slaclient/service/fed4fire/tests/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/sla/slaclient/service/fed4fire/tests/testparsejson.py b/sla/slaclient/service/fed4fire/tests/testparsejson.py new file mode 100755 index 00000000..7710f0ff --- /dev/null +++ b/sla/slaclient/service/fed4fire/tests/testparsejson.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- + +from unittest import TestCase +import datetime +import json + +from slaclient.service.fed4fire import jsonparser + + +class ParseJsonTestCase(TestCase): + + def setUp(self): + + self.from_json = None + + self.expirationtime = datetime.datetime.combine( + datetime.date.today(), + datetime.time(0, 0, 0) + ) + + self.template = dict( + template_id="template-id", + template_name="template-name", + provider="provider-id", + service_id="service-id", + expiration_time=self.expirationtime.isoformat(), + service_properties=[ + dict(servicename=None, name="uptime", metric=None, + location=None), + dict(servicename="service-name1", name="uptime", metric=None, + location=""), + dict(servicename="service-name2", name="metric1", + metric="xs:string", location=None), + dict(servicename="service-name2", name="metric2", + metric="xs:double", location="//monitoring/metric2") + ] + ) + + self.agreement = dict( + agreement_id="agreement-id", + template_id="template-id", + agreement_name="agreement-name", + consumer="consumer-id", + provider="provider-id", + service_id="service-id", + expiration_time=self.expirationtime.isoformat(), + guarantees=[ + dict(name="sin", bounds=(-1, 1)) + ] + ) + + def _check_dict(self, d, is_agreement): + o = self.from_json(json.dumps(d)) + self.assertEquals(d.get("template_id", None), o.template_id) + if is_agreement: + self.assertEquals(d.get("agreement_id"), o.agreement_id or None) + self.assertEquals(d.get("agreement_name"), o.agreement_name) + self.assertEquals(d.get("consumer"), o.consumer or None) + else: + self.assertEquals(d.get("template_name"), o.template_name or None) + self.assertEquals(d.get("provider"), o.provider) + self.assertEquals(d.get("service_id"), o.service_id) + self.assertEquals(d.get("expiration_time"), o.expiration_time_iso) + if "service_properties" in d: + for i in range(0, len(d["service_properties"])): + self.assertEquals( + d["service_properties"][i].get("servicename"), + o.service_properties[i].servicename + ) + self.assertEquals( + d["service_properties"][i].get("name"), + o.service_properties[i].name + ) + self.assertEquals( + d["service_properties"][i].get("metric"), + o.service_properties[i].metric + ) + self.assertEquals( + d["service_properties"][i].get("location"), + o.service_properties[i].location + ) + if "guarantees" in d: + for i in range(0, len(d["guarantees"])): + self.assertEquals( + d["guarantees"][i].get("name"), + o.guarantee_terms[i].metric_name + ) + self.assertEquals( + d["guarantees"][i].get("bounds"), + o.guarantee_terms[i].bounds + ) + + def test_template_from_json(self): + self.from_json = jsonparser.templateinput_from_json + + # + # Add fields one by one, and check + # + d = dict() + for key in self.template: + if key == "service_properties": + d[key] = [] + for prop in self.template[key]: + d[key].append(prop) + self._check_dict(d, False) + else: + d[key] = self.template[key] + self._check_dict(d, False) + + def test_agreement_from_json(self): + self.from_json = jsonparser.agreementinput_from_json + + # + # Add fields one by one, and check + # + d = dict() + + for key in self.agreement: + if key == "guarantees": + d[key] = [] + for term in self.agreement[key]: + d[key].append(term) + self._check_dict(d, True) + else: + d[key] = self.agreement[key] + self._check_dict(d, True) diff --git a/sla/slaclient/service/fed4fire/tests/testservice.py b/sla/slaclient/service/fed4fire/tests/testservice.py new file mode 100755 index 00000000..58a24f29 --- /dev/null +++ b/sla/slaclient/service/fed4fire/tests/testservice.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- + +import uuid +import json + +from unittest import TestCase +from slaclient.service.fed4fire import fed4fireservice +from slaclient import restclient +from slaclient.templates.fed4fire.django.factory import Factory as TemplateFactory + + +class Fed4FireServiceTestCase(TestCase): + + def setUp(self): + self.context = fed4fireservice.ServiceContext( + restclient.Factory("http://localhost:8080/sla-service"), + TemplateFactory() + ) + self.provider_id = str(uuid.uuid4()) + self.template_id = str(uuid.uuid4()) + self.provider = { + "uuid": self.provider_id, + "name": "provider-" + self.provider_id[0:4] + } + self.template = { + "template_id": self.template_id, + "template_name": "template-name", + "provider": self.provider_id, + "service_id": "service-test", + "expiration_time": "2014-03-28T13:55:00Z", + "service_properties": [ + {"name": "uptime"}, + {"name": "responsetime"} + ] + } + self.agreement = { + "agreement_id": str(uuid.uuid4()), + "template_id": self.template_id, + "expiration_time": "2014-03-28T13:55:00Z", + "consumer": "consumer-a", + "guarantees": [ + { + "name": "uptime", + "bounds": ["0.9", "1"] + } + ] + } + + def test(self): + self._test_provider() + self._test_template() + self._test_agreement() + + def _test_provider(self): + json_data = json.dumps(self.provider) + fed4fireservice.createprovider(json_data, self.context) + + def _test_template(self): + json_data = json.dumps(self.template) + fed4fireservice.createtemplate(json_data, self.context) + + def _test_agreement(self): + json_data = json.dumps(self.agreement) + fed4fireservice.createagreement(json_data, self.context) + +def main(): + context = fed4fireservice.ServiceContext( + restclient.Factory(), + TemplateFactory() + ) + provider_id = "trento" + template_id = "template_vm-Trento:193.205.211.xx" + provider = { + "uuid": provider_id, + "name": "provider-" + provider_id[0:4] + } + template = { + "template_id": template_id, + "template_name": "template-name", + "provider": provider_id, + "service_id": "service-test", + "expiration_time": "2014-03-28T13:55:00Z", + "service_properties": [ + {"name": "uptime"}, + {"name": "responsetime"} + ] + } + agreement = { + "agreement_id": str(uuid.uuid4()), + "template_id": template_id, + "expiration_time": "2014-03-28T13:55:00Z", + "consumer": "consumer-a", + # the provider id must be repeated + "provider": provider_id, + "guarantees": [ + { + "name": "uptime", + "bounds": ["0.9", "1"] + } + ] + } + + json_data = json.dumps(agreement) + fed4fireservice.createagreement(json_data, context) + + +if __name__ == "__main__": + main() + + diff --git a/sla/slaclient/templates/__init__.py b/sla/slaclient/templates/__init__.py new file mode 100755 index 00000000..0f2635e5 --- /dev/null +++ b/sla/slaclient/templates/__init__.py @@ -0,0 +1 @@ +from sla.slaclient.templates.templates import Template \ No newline at end of file diff --git a/sla/slaclient/templates/fed4fire/__init__.py b/sla/slaclient/templates/fed4fire/__init__.py new file mode 100755 index 00000000..950f42db --- /dev/null +++ b/sla/slaclient/templates/fed4fire/__init__.py @@ -0,0 +1,4 @@ +from sla.slaclient.templates.fed4fire.fed4fire import TemplateInput +from sla.slaclient.templates.fed4fire.fed4fire import AgreementInput +from sla.slaclient.templates.fed4fire.fed4fire import render_slaagreement +from sla.slaclient.templates.fed4fire.fed4fire import render_slatemplate diff --git a/sla/slaclient/templates/fed4fire/django/__init__.py b/sla/slaclient/templates/fed4fire/django/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/sla/slaclient/templates/fed4fire/django/agreement.xml b/sla/slaclient/templates/fed4fire/django/agreement.xml new file mode 100755 index 00000000..096a7e3c --- /dev/null +++ b/sla/slaclient/templates/fed4fire/django/agreement.xml @@ -0,0 +1,47 @@ + + + {% if data.agreement_name %}{{data.agreement_name}}{% endif %} + + + {{data.consumer}} + {{data.provider}} + AgreementResponder + {{data.expiration_time_iso}} + {{data.template_id}} + {% if data.service_id %}{{data.service_id}}{% endif %} + + + + + + + {% for property in data.service_properties %} + {{property.location|default:property.name}} + + {% endfor %} + + {% for term in data.guarantee_terms %} + + {# do not need servicescope #} + {% for scope in term.scopes %} + + {% endfor %} + + + {{term.servicelevelobjective.kpiname}} + + {% autoescape off %} + {{term.servicelevelobjective.customservicelevel}} + {% endautoescape %} + + + + {% endfor %} + + + \ No newline at end of file diff --git a/sla/slaclient/templates/fed4fire/django/factory.py b/sla/slaclient/templates/fed4fire/django/factory.py new file mode 100755 index 00000000..c7e43891 --- /dev/null +++ b/sla/slaclient/templates/fed4fire/django/factory.py @@ -0,0 +1,70 @@ +"""Django implementation of the templating needed in Fed4FIRE. +""" +import pkgutil +import django.template +from django.conf import settings +import sla.slaclient + +# +# Package where to read the template files +# +_package = "sla.slaclient.templates.fed4fire.django" + +# +# Filename of the sla-agreement template +# +_AGREEMENT_FILENAME = "agreement.xml" + +# +# Filename of the sla-template template +# +_TEMPLATE_FILENAME = "template.xml" + + +class Factory(object): + + def __init__(self): + self.slaagreement_tpl = None + self.slatemplate_tpl = None + + def _lazy_init(self): + if not settings.configured: + settings.configure() + + @staticmethod + def _read(filename): + string = pkgutil.get_data(_package, filename) + return string + + def _get_agreement_tpl(self): + self._lazy_init() + if self.slaagreement_tpl is None: + self.slaagreement_tpl = Factory._read(_AGREEMENT_FILENAME) + return self.slaagreement_tpl + + def _get_template_tpl(self): + self._lazy_init() + if self.slatemplate_tpl is None: + self.slatemplate_tpl = Factory._read(_TEMPLATE_FILENAME) + return self.slatemplate_tpl + + def slaagreement(self): + tpl = self._get_agreement_tpl() + result = Template(tpl) + return result + + def slatemplate(self): + tpl = self._get_template_tpl() + result = Template(tpl) + return result + + +class Template(sla.slaclient.templates.Template): + + def __init__(self, string): + self.impl = django.template.Template(string) + + def render(self, data): + context = django.template.Context(dict(data=data)) + result = self.impl.render(context) + return result diff --git a/sla/slaclient/templates/fed4fire/django/template.xml b/sla/slaclient/templates/fed4fire/django/template.xml new file mode 100755 index 00000000..659e44e7 --- /dev/null +++ b/sla/slaclient/templates/fed4fire/django/template.xml @@ -0,0 +1,28 @@ + + + {% if data.template_name %}{{data.template_name}}{% endif %} + + + {% if data.provider %}{{data.provider}}{% endif %} + AgreementResponder + {% if data.expiration_time %}{{data.expiration_time_iso}}{% endif %} + {{data.service_id}} + + + + + + + {% for property in data.service_properties %} + {{property.location|default:property.name}} + + {% endfor %} + + + + diff --git a/sla/slaclient/templates/fed4fire/fed4fire.py b/sla/slaclient/templates/fed4fire/fed4fire.py new file mode 100755 index 00000000..efb939c7 --- /dev/null +++ b/sla/slaclient/templates/fed4fire/fed4fire.py @@ -0,0 +1,293 @@ +# -*- coding: utf-8 -*- +"""Template system for xifi project. + +The specific template system is configured with the factory module variable. + +By default, it is set to use django. + +Each implementation must define a factory module/object, defining: +* slaagreement() +* slatemplate() + +that returns a slaclient.templates.Template-compliant object that performs +the actual render. + +This module defines two facade methods: +* render_slaagreement(data) +* render_slatemplate(data) + +and the corresponding input classes: +* AgreementInput +* TemplateInput + +Usage: + # Thread safe + import sla.slaclient.templates.fed4fire + data = sla.slaclient.templates.fed4fire.TemplateInput(template_id="template-test") + t = sla.slaclient.templates.fed4fire.django.Factory().slatemplate() + slatemplate_xml = t.render(data) + + # Non thread safe + import sla.slaclient.templates.fed4fire + data = sla.slaclient.templates.fed4fire.TemplateInput(template_id="template-test") + slatemplate_xml = sla.slaclient.templates.fed4fire.render_slatemplate(data) + +Notes about agreements in XiFi: + The ws-agreement specification does not address where to place the name/id + of the service (as known outside SLA) being defined in the + agreement/template xml. So, it has been defined an element + wsag:Context/sla:Service, whose text is the name/id of the service. This + is known here as serviceId. + + An agreement/template can represent zero or more than one existing services. + The guarantee terms, service description terms, etc, use the attribute + serviceName to reference (internally in the xml) the service. So, there + could be more than one serviceName in a xml (as opposed to the former + serviceId). In Xifi, there is only one service per agreement, so we + can give serviceId and serviceName the same value. + + A ServiceReference defines how a serviceName is known externally: a + service reference can be a name, a location, a structure containing both... + + The service properties are a set of variables that are used in the guarantee + terms contraints. So, for example, if a constraint is : "uptime < 90", we + can have 2 service properties: ActualUptime and DesiredUptime. And the + constraint will be "ActualUptime < DesiredUptime". This is the theory. But + we're not going to use the service properties this way. We will not use the + thresholds as service properties; only the actual metric. So, in this case, + the service property is defined in ws-agreement as: + + + service-ping/Uptime + + + The "location" is the strange value here. Ws-agreement says that it is a + "structural reference" to the place where to find the actual value of the + metric. The examples I've found are references to the + ServiceDescriptionTerms in the agreement itself. We are not using SDTs + (they are used to describe the service to be instantiated), so we can + extrapolate the location as the "abstract location of the metric". + + In summary, in XiFi, the service properties will hold the metrics being + monitored for a service. + + And the guarantee terms hold the constraints that are being enforced for + the service in this agreement (maybe we are only interested in enforcing + one of the metrics). + + A guarantee term is defined as: + + + + + Uptime + + {"constraint" : "Uptime BETWEEN (90, 100)"} + + + + + + * Name is a name for the guarantee term. In Xifi, the name will have the + value "GT_" + * ServiceName is an internal reference in the agreement to the service + being enforced, as an agreement can created for more than one service. + In Xifi, to my knowledge, one service: one agreement, so this service + name is not really important. + * KpiName is a name given to the constraint, and I am using the same name + as the service property used in the constraint. This makes more sense + when using thresholds as service properties (e.g., a kpi called + "uptime" could be defined as : + "actual_uptime BETWEEN(lower_uptime, upper_uptime)"). + + The CustomServiceLevel is not specified by ws-agreement, so it's something + to be defined by the implementation. + +""" + +from sla.slaclient import wsag_model +import pdb + +from sla.slaclient.templates.fed4fire.django.factory import Factory +factory = Factory() + + +def _getfactory(): + # + # Hardwired above to avoid multheading issues. This will need some + # refactoring if the factory really needs to be configurable. + # + + global factory + #if factory is None: + # from slaclient.templates.fed4fire.django.factory import Factory + # factory = Factory() + return factory + + +def render_slaagreement(data): + """Generate a sla agreement based on the supplied data. + + :type data: AgreementInput + """ + print "render_slaagreement" + template = _getfactory().slaagreement() + #pdb.set_trace() + rendered = template.render(data) + return rendered + + +def render_slatemplate(data): + """Generate a sla template based on the supplied data. + + :type data: TemplateInput + """ + template = _getfactory().slatemplate() + return template.render(data) + + +class TemplateInput(object): + + def __init__(self, + template_id="", + template_name="", + provider="", + service_id="", + expiration_time=None, + service_properties=()): + """Input data to the template for generating a sla-template. + + :param str template_id: optional TemplateId. If not specified, the + SlaManager should provide one. + :param str template_name: optional name for the template. + :param str service_id: Domain id/name of the service. + :param str provider: optional Resource Id of the provider party in the + agreement. The provider must exist previously in the SlaManager. + :param expiration_time: optional expiration time of this template. + :type expiration_time: datetime.datetime + :param service_properties: Metrics that the provider is able to + monitor for this service. + :type service_properties: list[slaclient.wsag_model.Agreement.Property] + """ + self.template_id = template_id + self.template_name = template_name + self.service_id = service_id + self.provider = provider + self.expiration_time = expiration_time + self.expiration_time_iso = \ + expiration_time.isoformat() if expiration_time else None + self.service_properties = service_properties + + def __repr__(self): + s = "" + return s.format( + self.template_id, + self.template_name, + self.service_id, + self.provider, + self.expiration_time_iso, + repr(self.service_properties) + ) + + +class AgreementInput(object): + + class GuaranteeTerm(object): + + def __init__(self, + metric_name="", + bounds=(0, 0)): + """Creates a GuaranteeTerm. + + Take into account that the GT's name is based on the metric_name. + :param str metric_name: name of the service property being enforced. + :param bounds: (lower, upper) bounds of the metric values. + :type bounds: (float, float) + """ + self.name = "GT_{}".format(metric_name) + self.metric_name = metric_name + self.kpiname = metric_name + self.bounds = bounds + + def __init__(self, + agreement_id="", + agreement_name="", + service_id="", + consumer="", + provider="", + template_id="", + expiration_time=None, + service_properties=(), + guarantee_terms=()): + """Input data to the template for generating a sla-agreement + + :param str agreement_id: optional agreement id. If not supplied, + the SlaManager should create one. + :param str agreement_name: optional agreement name + :param str service_id: Domain id/name of the service. + :param str consumer: Id of the consumer party in the agreement. + :param str provider: Resource Id of the provider party in the agreement. + The provider must exist previously in the SlaManager. + :param str template_id: TemplateId of the template this agreement is + based on. + :param expiration_time: Expiration time of this agreement. + :type expiration_time: datetime.datetime + :param service_properties: Should be the same of the template. + :type service_properties: list[slaclient.wsag_model.Agreement.Property] + :param guarantee_terms: Guarantee terms to be enforced in this + agreement. + :type guarantee_terms: list(AgreementInput.GuaranteeTerm) + """ + self.agreement_id = agreement_id + self.agreement_name = agreement_name + self.service_id = service_id + self.consumer = consumer + self.provider = provider + self.template_id = template_id + self.expiration_time = expiration_time + self.expiration_time_iso = \ + expiration_time.isoformat() if expiration_time else None + self.service_properties = service_properties + self.guarantee_terms = guarantee_terms + + def __repr__(self): + s = "" + return s.format( + self.agreement_id, + self.agreement_name, + self.service_id, + self.consumer, + self.provider, + self.template_id, + self.expiration_time, + repr(self.service_properties), + repr(self.guarantee_terms) + ) + + def from_template(self, slatemplate): + """Return a new agreement based on this agreement and copying info + (overriding if necessary) from a slatemplate. + + :type slatemplate: wsag_model.Template + :rtype: AgreementInput + """ + # + # NOTE: templateinput does not address guaranteeterms (yet) + # + result = AgreementInput( + agreement_id=self.agreement_id, + agreement_name=self.agreement_name, + service_id=slatemplate.context.service, + consumer=self.consumer, + provider=slatemplate.context.provider or self.provider, + template_id=slatemplate.template_id, + expiration_time=self.expiration_time, + service_properties=slatemplate.variables.values(), + guarantee_terms=slatemplate.guaranteeterms.values() + ) + print result.guarantee_terms[0] + return result diff --git a/sla/slaclient/templates/fed4fire/tests/__init__.py b/sla/slaclient/templates/fed4fire/tests/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/sla/slaclient/templates/fed4fire/tests/testtemplates.py b/sla/slaclient/templates/fed4fire/tests/testtemplates.py new file mode 100755 index 00000000..64d0fc86 --- /dev/null +++ b/sla/slaclient/templates/fed4fire/tests/testtemplates.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- + +from unittest import TestCase +import datetime + +from slaclient import wsag_model +from slaclient import xmlconverter +import slaclient.templates.fed4fire +from slaclient.templates.fed4fire import TemplateInput +from slaclient.templates.fed4fire import AgreementInput + + + + +class TemplatesTestCase(TestCase): + + def setUp(self): + self.converter = xmlconverter.AgreementConverter() + + self.expirationtime = datetime.datetime.combine( + datetime.date.today(), + datetime.time(0, 0, 0) + ) + self.templateinput = TemplateInput( + template_id="template-id", + template_name="template-name", + service_id="service-name", + expiration_time=self.expirationtime, + service_properties=[ + wsag_model.Agreement.Property( + name="uptime", + metric="xs:double", + location="uptime"), + wsag_model.Agreement.Property( + name="responsetime", + location="responsetime"), + wsag_model.Agreement.Property( + name="quality", + metric="xs:string"), + ] + ) + self.agreementinput = AgreementInput( + agreement_id="agreement-id", + agreement_name="agreement-name", + consumer="consumer-id", + provider="provider-id", + service_id="service-name", + template_id="template-id", + expiration_time=self.expirationtime, + service_properties=self.templateinput.service_properties, + guarantee_terms=[ + AgreementInput.GuaranteeTerm( + "uptime", (0.9, 1) + ), + AgreementInput.GuaranteeTerm( + "responsetime", (0, 200) + ) + ] + ) + + def test_template(self): + slatemplate = slaclient.templates.fed4fire.render_slatemplate( + self.templateinput + ) + # convert xml to wsag_model classes + actual = xmlconverter.convertstring(self.converter, slatemplate) + """:type: wsag_model.Template""" + + expected = self.templateinput + + self.assertEquals( + expected.template_id, + actual.template_id + ) + self._check_common(expected, actual) + print slatemplate + + def test_agreement(self): + slaagreement = slaclient.templates.fed4fire.render_slaagreement( + self.agreementinput + ) + # convert xml to wsag_model classes + actual = xmlconverter.convertstring(self.converter, slaagreement) + """:type: wsag_model.Agreement""" + + expected = self.agreementinput + + self.assertEquals( + expected.agreement_id, + actual.agreement_id + ) + expected.consumer and self.assertEquals( + expected.consumer, + actual.context.consumer + ) + self._check_common(expected, actual) + self._check_guarantee_terms(expected, actual) + print slaagreement + + def _check_common(self, expected, actual): + if expected.provider: + self.assertEquals( + expected.provider, + actual.context.provider + ) + self.assertEquals( + expected.expiration_time_iso, + actual.context.expirationtime + ) + self.assertEquals( + expected.service_id, + actual.context.service + ) + self._check_properties(expected, actual) + + def _check_properties(self, expected, actual): + for expected_prop in expected.service_properties: + actual_prop = actual.variables[expected_prop.name] + self.assertEquals( + expected_prop.name, + actual_prop.name + ) + self.assertEquals( + expected_prop.location or expected_prop.name, + actual_prop.location + ) + self.assertEquals( + expected_prop.metric or 'xs:double', + actual_prop.metric + ) + + def _check_guarantee_terms(self, expected, actual): + """ + :type expected: AgreementInput + :type actual: wsag_model.Agreement + """ + for expected_term in expected.guarantee_terms: + actual_term = actual.guaranteeterms[expected_term.name] + + if actual_term is None: + self.assertEquals(expected_term.name, None) + self.assertEquals( + expected_term.kpiname, + actual_term.servicelevelobjective.kpiname + ) + diff --git a/sla/slaclient/templates/templates.py b/sla/slaclient/templates/templates.py new file mode 100755 index 00000000..345a2cfc --- /dev/null +++ b/sla/slaclient/templates/templates.py @@ -0,0 +1,31 @@ +"""This module and submodules offers a generic way to create ws-agreement +representations from structured data, by using templates. + +Each submodule (corresponding to a project) is responsible to declare +the structured data to be used as input, and handle the specific template +library. + +This module only defines a sample interface to be used for each Template object +used by each project. + +Sample usage (read specific submodules' docs): +data = slaclient..Input(...) +tpl = slaclient..Template(...) +tpl.render(data) + +""" + + +class Template(object): + + def __init__(self, file_): + """This is the interface that all project templates should "implement". + + It mimics the behavior of django templates. + """ + pass + + def render(self, data): + """Renders this template using 'data' as input. + """ + pass diff --git a/sla/slaclient/tests/__init__.py b/sla/slaclient/tests/__init__.py new file mode 100755 index 00000000..086be4b7 --- /dev/null +++ b/sla/slaclient/tests/__init__.py @@ -0,0 +1 @@ +__author__ = 'a565180' diff --git a/sla/slaclient/tests/agreement.xml b/sla/slaclient/tests/agreement.xml new file mode 100755 index 00000000..20cc05d3 --- /dev/null +++ b/sla/slaclient/tests/agreement.xml @@ -0,0 +1,81 @@ + + + + ExampleAgreement + + RandomClient + provider-prueba + + AgreementResponder + 2014-03-07T12:00:00 + contract-template-2007-12-04 + ExampleService + + + + + + DSL expression + + + DSL expression + + + + + + + + + qos:ResponseTime + + + qos:Performance + + + + + /operation1 + /operation2 + + + + + ResponseTime + {"constraint" : "ResponseTime BETWEEN (0,0.9)"} + + + + + + + + Performance + {"constraint" : "Performance BETWEEN (0.1,1)"} + + + + 3 + + + 10 + + EUR + 99 + + + + + + + + + + diff --git a/sla/slaclient/tests/testconverters.py b/sla/slaclient/tests/testconverters.py new file mode 100755 index 00000000..82b6aa79 --- /dev/null +++ b/sla/slaclient/tests/testconverters.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +from unittest import TestCase +from pprint import pprint +import json + +from slaclient import wsag_model +from slaclient import xmlconverter + + +class AgreementAnnotatorTestCase(TestCase): + + def setUp(self): + self.violation = """ + + ce0e148f-dfac-4492-bb26-ad2e9a6965ec + agreement04 + + Performance + 2014-01-14T11:28:22Z + 0.09555700123360344 + """ + + self.provider = """ + + 1ad9acb9-8dbc-4fe6-9a0b-4244ab6455da + Provider2 + """ + + self.list = """ + + + + 1ad9acb9-8dbc-4fe6-9a0b-4244ab6455da + Provider1 + + + 2ad9acb9-8dbc-4fe6-9a0b-4244ab6455da + Provider2 + + + """ + + self.agreement_status = """ + { + "AgreementId":"agreement03", + "guaranteestatus":"VIOLATED", + "guaranteeterms": + [ + {"name":"GT_ResponseTime","status":"FULFILLED"}, + {"name":"GT_Performance","status":"VIOLATED"} + ] + }""" + + def test_agreement(self): + conv = xmlconverter.AgreementConverter() + + out = xmlconverter.convertfile(conv, "slagui/testing/agreement.xml") + """:type : Agreement""" + + #pprint(out) + + def test_provider(self): + conv = xmlconverter.ProviderConverter() + out = xmlconverter.convertstring(conv, self.provider) + #pprint(out) + + def test_violation(self): + conv = xmlconverter.ViolationConverter() + out = xmlconverter.convertstring(conv, self.violation) + #pprint(out) + + def test_list(self): + conv = xmlconverter.ListConverter(xmlconverter.ProviderConverter()) + out = xmlconverter.convertstring(conv, self.list) + #pprint(out) + + def test_agreement_status_decode(self): + json_obj = json.loads(self.agreement_status) + out = wsag_model.AgreementStatus.json_decode(json_obj) + #pprint(out) diff --git a/sla/slaclient/wsag_model.py b/sla/slaclient/wsag_model.py new file mode 100755 index 00000000..1c9bd41a --- /dev/null +++ b/sla/slaclient/wsag_model.py @@ -0,0 +1,233 @@ +from datetime import datetime + +"""Contains the bean models for the SlaManager xml/json types +""" + + +class Agreement(object): + + class Context(object): + def __init__(self): + self.expirationtime = datetime.now() + self.service = "" + self.initiator = "" + self.responder = "" + self.provider = "" + self.consumer = "" + + def __repr__(self): + s = "" + return s.format( + repr(self.expirationtime), + repr(self.provider), + repr(self.consumer), + repr(self.service)) + + def service_formatted(self): + return self.service.replace('_', ' ') + + def testbed_formatted(self): + return self.template_id.replace('Service', ' - ') + + class Property(object): + def __init__(self): + self.servicename = "" + self.name = "" + self.metric = "" + self.location = "" + + def __repr__(self): + str_ = "" + return str_.format( + repr(self.name), + repr(self.servicename), + repr(self.metric), + repr(self.location)) + + class GuaranteeTerm(object): + + class GuaranteeScope(object): + def __init__(self): + self.servicename = "" + self.scope = "" + + def __repr__(self): + return ")".format( + repr(self.servicename), + repr(self.scope) + ) + + class ServiceLevelObjective(object): + def __init__(self): + self.kpiname = "" + self.customservicelevel = "" + + def __repr__(self): + s = "" + return s.format( + repr(self.kpiname), + repr(self.customservicelevel) + ) + + def __init__(self): + self.name = "" + self.scopes = [] # item: GuaranteeScope + """:type : list[Agreement.GuaranteeTerm.GuaranteeScope]""" + self.servicelevelobjective = \ + Agreement.GuaranteeTerm.ServiceLevelObjective() + + def __repr__(self): + s = "" + return s.format( + repr(self.scopes), + repr(self.servicelevelobjective) + ) + + def __init__(self): + """Simple bean model for a ws-agreement agreement/template + """ + self.context = Agreement.Context() + self.agreement_id = "" + self.descriptionterms = {} + self.variables = {} # key: Property.name / value: Property + """:type : dict[str,Agreement.Property]""" + self.guaranteeterms = {} # key: GT.name / value: GT + """:type : dict[str,Agreement.GuaranteeTerm]""" + + def __repr__(self): + s = ("") + return s.format( + repr(self.agreement_id), + repr(self.context), + repr(self.descriptionterms), + repr(self.variables), + repr(self.guaranteeterms) + ) + + +class Template(Agreement): + #egarrido this code has been copied from xifi and has not beeing tested + def __init__(self): + super(Template, self).__init__() + self.template_id = "" + + def __repr__(self): + s = ("") + return s.format( + repr(self.template_id), + repr(self.context), + repr(self.descriptionterms), + repr(self.variables), + repr(self.guaranteeterms) + ) + + +class Enforcement(object): + def __init__(self): + """Simple bean model for an enforcement""" + self.agreement_id = "" + self.enabled = "" + + def __repr__(self): + return ("".format( + self.agreement_id, + self.enabled) + ) + +class AgreementStatus(object): + + class StatusEnum: + VIOLATED = "VIOLATED" + FULFILLED = "FULFILLED" + NON_DETERMINED = "NON_DETERMINED" + + class GuaranteeTermStatus(object): + def __init__(self): + self.name = "" + self.status = "" + + def __repr__(self): + s = "" + return s.format(self.name, self.status) + + def __init__(self): + self.agreement_id = "" + self.guaranteestatus = "" + self.guaranteeterms = [] + + def __repr__(self): + return ( + "").format( + self.agreement_id, + self.guaranteestatus, + repr(self.guaranteeterms)) + + @staticmethod + def json_decode(json_obj): + o = AgreementStatus() + o.agreement_id = json_obj["AgreementId"] + o.guaranteestatus = json_obj["guaranteestatus"] + + for term in json_obj["guaranteeterms"]: + t = AgreementStatus.GuaranteeTermStatus() + t.name = term["name"] + t.status = term["status"] + o.guaranteeterms.append(t) + return o + + +class Violation(object): + def __init__(self): + """Simple bean model for a violation""" + self.uuid = "" + self.contract_uuid = "" + self.service_scope = "" + self.metric_name = "" + self.datetime = datetime.now() + self.actual_value = 0 + + def __repr__(self): + return ("".format( + self.uuid, + self.contract_uuid, + self.service_scope, + self.metric_name, + self.datetime, + self.actual_value) + ) + + +class Provider(object): + def __init__(self): + """Simple bean model for a provider""" + self.uuid = "" + self.name = "" + + def __repr__(self): + return ("".format( + self.uuid, + self.name) + ) + def to_xml(self): + xml = "{}{}""".format( + self.uuid, + self.name + ) + return xml + + @staticmethod + def from_dict(d): + """Creates a Provider object from a dict structure (e.g. + a deserialized json string) + + Usage: + json_obj = json.loads(json_data) + out = wsag_model.Provider.from_dict(json_obj) + """ + result = Provider(d["uuid"], d["name"]) + return result diff --git a/sla/slaclient/xmlconverter.py b/sla/slaclient/xmlconverter.py new file mode 100755 index 00000000..831df53c --- /dev/null +++ b/sla/slaclient/xmlconverter.py @@ -0,0 +1,364 @@ +# -*- coding: utf-8 -*- + +"""Converts from XML to objects for ws-agreement agreements/templates or any +other xml returned by SLA Manager. + +This module offers a set of converters from xml formats returned by SLA Manager +to a more-friendly POJO instances. + +The converters are designed to be pluggable: see ListConverter. + + +Usage: +c = AnyConverter() or +c = ListConverter(AnyOtherConverter()) + +convertstring(c, "") + +convertfile(c, "file.xml") + +root = ElementTree.parse("file.xml") +c.convert(root.getroot()) + +""" + +from xml.etree import ElementTree +from xml.etree.ElementTree import Element +import dateutil.parser + +from wsag_model import Agreement +from wsag_model import Template +from wsag_model import Violation +from wsag_model import Provider +from wsag_model import Enforcement + + +def convertfile(converter, f): + """Reads and converts a xml file + + :rtype : object + :param Converter converter: + :param str f: file to read + """ + tree = ElementTree.parse(f) + result = converter.convert(tree.getroot()) + return result + + +def convertstring(converter, string): + """Converts a string + + :rtype : object + :param Converter converter: + :param str string: contains the xml to convert + """ + root = ElementTree.fromstring(string) + result = converter.convert(root) + return result + + +class Converter(object): + + def __init__(self): + """Base class for converters + """ + pass + + def convert(self, xmlroot): + """Converts the given xml in an object + + :rtype : Object that represents the xml + :param Element xmlroot: root element of xml to convert. + """ + return None + + +class ListConverter(Converter): + def __init__(self, innerconverter): + super(ListConverter, self).__init__() + self.innerconverter = innerconverter + + def convert(self, xmlroot): + result = [] + + for item in xmlroot.find("items"): # loop through "items" children + inner = self.innerconverter.convert(item) + result.append(inner) + return result + + +class ProviderConverter(Converter): + """Converter for a provider. + + Input: + + 1ad9acb9-8dbc-4fe6-9a0b-4244ab6455da + Provider2 + + + Output: + wsag_model.Provider + """ + + def __init__(self): + super(ProviderConverter, self).__init__() + + def convert(self, xmlroot): + result = Provider() + result.uuid = xmlroot.find("uuid").text + result.name = xmlroot.find("name").text + return result + + +class EnforcementConverter(Converter): + """Converter for an Enforcement job. + + Input: + + agreement03 + false + + + Output: + wsag_model.Enforcement + """ + + def __init__(self): + super(EnforcementConverter, self).__init__() + + def convert(self, xmlroot): + result = Enforcement() + result.agreement_id = xmlroot.find("agreement_id").text + result.enabled = xmlroot.find("enabled").text + return result + +class ViolationConverter(Converter): + """Converter for a violation. + + Input: + + ce0e148f-dfac-4492-bb26-ad2e9a6965ec + agreement04 + + Performance + 2014-01-14T11:28:22Z + 0.09555700123360344 + + + Output: + wsag_model.Violation + """ + def __init__(self): + super(ViolationConverter, self).__init__() + + def convert(self, xmlroot): + result = Violation() + result.uuid = xmlroot.find("uuid").text + result.contract_uuid = xmlroot.find("contract_uuid").text + result.service_scope = xmlroot.find("service_scope").text + result.metric_name = xmlroot.find("metric_name").text + result.actual_value = xmlroot.find("actual_value").text + dt_str = xmlroot.find("datetime").text + result.datetime = dateutil.parser.parse(dt_str) + return result + + +class AgreementConverter(Converter): + def __init__(self): + """Converter for an ws-agreement agreement or template. + """ + super(AgreementConverter, self).__init__() + self._namespaces = { + "wsag": "http://www.ggf.org/namespaces/ws-agreement", + "sla": "http://sla.atos.eu", + "xifi": "http://sla.xifi.eu" + } + self.agreement_tags = ( + "{{{}}}Agreement".format(self._namespaces["wsag"]), + ) + self.template_tags = ( + "{{{}}}Template".format(self._namespaces["wsag"]), + ) + + def convert(self, xmlroot): + """ + :param Element xmlroot: root element of xml to convert. + :rtype: wsag_model.Agreement + """ + if xmlroot.tag in self.agreement_tags: + result = Agreement() + result.agreement_id = xmlroot.attrib["AgreementId"] + elif xmlroot.tag in self.template_tags: + result = Template() + result.template_id = xmlroot.attrib["TemplateId"] + else: + raise ValueError("Not valid root element name: " + xmlroot.tag) + + context = xmlroot.find("wsag:Context", self._namespaces) + result.context = self._parse_context(context) + + terms = xmlroot.find("wsag:Terms/wsag:All", self._namespaces) + + properties = terms.findall("wsag:ServiceProperties", self._namespaces) + result.variables = self._parse_properties(properties) + + guarantees = terms.findall("wsag:GuaranteeTerm", self._namespaces) + result.guaranteeterms = self._parse_guarantees(guarantees) + + return result + + def _parse_context(self, element): + nss = self._namespaces + result = Agreement.Context() + + result.template_id = self._find_text(element, "wsag:TemplateId") + result.expirationtime = self._find_text(element, "wsag:ExpirationTime") + + service_elem = element.find("sla:Service", nss) + result.service = \ + service_elem.text if service_elem is not None else "" + + initiator = self._find_text(element, "wsag:AgreementInitiator") + responder = self._find_text(element, "wsag:AgreementResponder") + serviceprovider_elem = self._find_text(element, "wsag:ServiceProvider") + + # + # Deloop the initiator-responder indirection. + # + if serviceprovider_elem == "AgreementResponder": + consumer = initiator + provider = responder + elif serviceprovider_elem == "AgreementInitiator": + consumer = responder + provider = initiator + else: + raise ValueError( + "Invalid value for wsag:ServiceProvider : " + + serviceprovider_elem) + + result.initiator = initiator + result.responder = responder + result.provider = provider + result.consumer = consumer + + return result + + def _parse_property(self, element, servicename): + nss = self._namespaces + + key = _get_attribute(element, "Name") + value = Agreement.Property() + value.servicename = servicename + value.name = key + value.metric = _get_attribute(element, "Metric") + value.location = element.find("wsag:Location", nss).text + + return key, value + + def _parse_properties(self, elements): + result = {} + nss = self._namespaces + for element in elements: + servicename = _get_attribute(element, "ServiceName") + for var in element.findall("wsag:Variables/wsag:Variable", nss): + key, value = self._parse_property(var, servicename) + result[key] = value + + return result + + def _parse_guarantee_scope(self, element): + result = Agreement.GuaranteeTerm.GuaranteeScope() + result.servicename = _get_attribute(element, "ServiceName") + result.scope = element.text + return result + + def _parse_guarantee_scopes(self, elements): + result = [] + for scope in elements: + result.append(self._parse_guarantee_scope(scope)) + return result + + def _parse_guarantee(self, element): + nss = self._namespaces + + result = Agreement.GuaranteeTerm() + name = _get_attribute(element, "Name") + result.name = name + scopes = element.findall("wsag:ServiceScope", nss) + result.scopes = self._parse_guarantee_scopes(scopes) + + kpitarget = element.find( + "wsag:ServiceLevelObjective/wsag:KPITarget", nss) + slo = Agreement.GuaranteeTerm.ServiceLevelObjective() + result.servicelevelobjective = slo + slo.kpiname = kpitarget.find("wsag:KPIName", nss).text + slo.customservicelevel = kpitarget.find( + "wsag:CustomServiceLevel", nss).text + + return name, result + + def _parse_guarantees(self, elements): + + result = {} + for element in elements: + key, value = self._parse_guarantee(element) + result[key] = value + return result + + def _find_text(self, src, path): + """Returns the inner text of the element located in path from the src + element; None if no elements were found. + + :type src: Element + :type path: src + :rtype: str + + Usage: + text = _find_text(root, "wsag:Context/ExpirationTime") + """ + dst = src.find(path, self._namespaces) + if dst is None: + return "" + return dst.text + + +def _get_attribute(element, attrname): + """ + Get attribute from an element. + + Wrapper over Element.attrib, as this doesn't fallback to the element + namespace if the attribute is qnamed and the requested attribute name + is not. + + Ex: + + + _get_attribute(elem, "attr1") -> value1 + _get_attribute(elem, "attr2") -> value2 + _get_attribute(elem, "{uri}:attr1") -> Error + _get_attribute(elem, "{uri}:attr2") -> value2 + """ + isns = (attrname[0] == '{') + + # + # Handle qnamed request: + # attrname = {uri}name + # + if isns: + return element.attrib[attrname] + + # + # Handle non-qnamed request and non-qnamed actual_attr + # attrname = name + # actual_attr = name + # + if attrname in element.attrib: + return element.attrib[attrname] + + # + # Handle non-qnamed request but qnamed actualAttr + # attrname = name + # actual_attr = {uri}name + # + tag_uri = element.tag[0: element.tag.find('}') + 1] + return element.attrib[tag_uri + attrname] diff --git a/sla/slicetabsla.py b/sla/slicetabsla.py new file mode 100755 index 00000000..42718581 --- /dev/null +++ b/sla/slicetabsla.py @@ -0,0 +1,356 @@ +# this somehow is not used anymore - should it not be ? +from django.template import RequestContext +from django.shortcuts import render_to_response +from django.shortcuts import render +from django import forms + +from unfold.loginrequired import FreeAccessView +from unfold.page import Page +from sla.slaclient import restclient +from sla.slaclient import wsag_model +import wsag_helper +from myslice.theme import ThemeView +# from sla import SLAPlugin +from django.core.urlresolvers import reverse +from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger + +import slaclient.service.fed4fire.fed4fireservice as fed4fireservice +from rest_framework.views import APIView +from django.http import HttpResponse + +import json +import traceback + + +class Rol: + CONSUMER = "CONSUMER" + PROVIDER = "PROVIDER" + + +class AgreementsFilter(object): + def __init__(self, status=None, provider=None, consumer=None): + self.status = status + self.provider = provider + self.consumer = consumer + + def __repr__(self): + return "".format( + self.status, self.provider, self.consumer + ) + + @staticmethod + def _check(expectedvalue, actualvalue): + if expectedvalue is None or expectedvalue == '': + return True + else: + return actualvalue == expectedvalue + + def check(self, agreement): + """Check if this agreement satisfy the filter. + + The agreement must be previously annotated + """ + guaranteestatus = agreement.guaranteestatus + provider = agreement.context.provider + consumer = agreement.context.consumer + return ( + AgreementsFilter._check(self.status, guaranteestatus) and + AgreementsFilter._check(self.provider, provider) and + AgreementsFilter._check(self.consumer, consumer) + ) + + +class FilterForm(forms.Form): + _attrs = {'class': 'form-control'} + exclude = () + status = forms.ChoiceField( + choices=[ + ('', 'All'), + (wsag_model.AgreementStatus.StatusEnum.FULFILLED, 'Fulfilled'), + (wsag_model.AgreementStatus.StatusEnum.VIOLATED, 'Violated'), + (wsag_model.AgreementStatus.StatusEnum.NON_DETERMINED, 'Non determined')], + widget=forms.Select(attrs=_attrs), + required=False + ) + provider = forms.CharField( + widget=forms.TextInput(attrs=_attrs), + required=False + ) + consumer = forms.CharField( + widget=forms.TextInput(attrs=_attrs), + required=False + ) + + +class SLAView (FreeAccessView, ThemeView): + template_name = 'slice-tab-sla.html' + + def get (self, request, slicename, state=None): + + page=Page(request) + + consumer_id = None + agreement_id = None + enforcements = {} + + filter_ = None + form = FilterForm(request.GET) + if form.is_valid(): + print "IS VALID" + filter_ = _get_filter_from_form(form) + + consumer_id = _get_consumer_id(request) + + agreements = _get_agreements(agreement_id, consumer_id=consumer_id, filter_=filter_) + + for agreement in agreements: + enf = _get_enforcement(agreement.agreement_id) + enforcements[agreement.agreement_id] = enf.enabled + + for key, value in enforcements.items(): + print key + ": " + value + + template_env = {} + # write something of our own instead + # more general variables expected in the template + template_env['title'] = 'SLA Agreements' + template_env['agreements'] = agreements + template_env['username'] = request.user + template_env['slicename'] = slicename + template_env['enforcements'] = enforcements + + # the prelude object in page contains a summary of the requirements() for all plugins + # define {js,css}_{files,chunks} + prelude_env = page.prelude_env() + template_env.update(prelude_env) + + return render_to_response (self.template_name, template_env, context_instance=RequestContext(request)) + + +class AgreementsFilter(object): + def __init__(self, status=None, provider=None, consumer=None): + self.status = status + self.provider = provider + self.consumer = consumer + + def __repr__(self): + return "".format( + self.status, self.provider, self.consumer + ) + + @staticmethod + def _check(expectedvalue, actualvalue): + if expectedvalue is None or expectedvalue == '': + return True + else: + return actualvalue == expectedvalue + + def check(self, agreement): + """Check if this agreement satisfy the filter. + + The agreement must be previously annotated + """ + guaranteestatus = agreement.guaranteestatus + provider = agreement.context.provider + consumer = agreement.context.consumer + return ( + AgreementsFilter._check(self.status, guaranteestatus) and + AgreementsFilter._check(self.provider, provider) and + AgreementsFilter._check(self.consumer, consumer) + ) + + +class ContactForm(forms.Form): + subject = forms.CharField(max_length=100) + message = forms.CharField() + sender = forms.EmailField() + cc_myself = forms.BooleanField(required=False) + + +def _get_agreements_client(): + return restclient.Factory.agreements() + + +def _get_violations_client(): + return restclient.Factory.violations() + +def _get_enforcements_client(): + return restclient.Factory.enforcements() + +def _get_consumer_id(request): + return request.user + + +def _get_agreement(agreement_id): + + agreements_client = _get_agreements_client() + agreement, response = agreements_client.getbyid(agreement_id) + return agreement + +def _get_enforcement(agreement_id): + + enforcements_client = _get_enforcements_client() + enforcement, response = enforcements_client.getbyagreement(agreement_id) + return enforcement + +def _get_filter_from_form(form): + + data = form.cleaned_data + result = AgreementsFilter( + data["status"], data["provider"], data["consumer"]) + return result + +def agreement_term_violations(request, agreement_id, guarantee_name): + + page = Page(request) + prelude_env = page.prelude_env() + + annotator = wsag_helper.AgreementAnnotator() + agreement = _get_agreement(agreement_id) + violations = _get_agreement_violations(agreement_id, guarantee_name) + annotator.annotate_agreement(agreement) + + slicename = request.POST.get('slicename') + + paginator = Paginator(violations, 25) # Show 25 violations per page + page_num = request.GET.get('page') + + try: + violation_page = paginator.page(page_num) + except PageNotAnInteger: + # If page is not an integer, deliver first page. + violation_page = paginator.page(1) + except EmptyPage: + # If page is out of range (e.g. 9999), deliver first page. + violation_page = paginator.page(1) + + context = { + 'agreement_id': agreement_id, + 'guarantee_term': agreement.guaranteeterms[guarantee_name], + 'violations': violation_page, + 'agreement': agreement, + 'slicename': slicename, + } + + context.update(prelude_env) + + return render_to_response ('violations_template.html', context, context_instance=RequestContext(request)) +# return render(request, 'violations_template.html', context) + +def agreement_details(request, agreement_id): + + page = Page(request) + prelude_env = page.prelude_env() + + annotator = wsag_helper.AgreementAnnotator() + agreement = _get_agreement(agreement_id) + violations = _get_agreement_violations(agreement_id) + status = _get_agreement_status(agreement_id) + annotator.annotate_agreement(agreement, status, violations) + + violations_by_date = wsag_helper.get_violations_bydate(violations) + context = { + 'agreement_id': agreement_id, + 'agreement': agreement, + 'status': status, + 'violations_by_date': violations_by_date + } + + context.update(prelude_env) + + return render_to_response ('violations_template.html', context, context_instance=RequestContext(request)) + #return render(request, 'agreement_detail.html', context) + + +def _get_agreements_client(): + return restclient.Factory.agreements() + + +def _get_agreement(agreement_id): + + agreements_client = _get_agreements_client() + agreement, response = agreements_client.getbyid(agreement_id) + return agreement + +def _get_agreements(agreement_id, provider_id=None, consumer_id=None, filter_=None): + + agreements_client = _get_agreements_client() + if agreement_id is None: + if consumer_id is not None: + agreements, response = agreements_client.getbyconsumer(consumer_id) + elif provider_id is not None: + agreements, response = agreements_client.getbyprovider(provider_id) + else: + raise ValueError( + "Invalid values: consumer_id and provider_id are None") + else: + agreement, response = agreements_client.getbyid(agreement_id) + agreements = [agreement] + + annotator = wsag_helper.AgreementAnnotator() + for agreement in agreements: + id_ = agreement.agreement_id + status = _get_agreement_status(id_) + annotator.annotate_agreement(agreement, status) + + if filter_ is not None: + print "FILTERING ", repr(filter_) + agreements = filter(filter_.check, agreements); + else: + print "NOT FILTERING" + return agreements + + +def _get_agreements_by_consumer(consumer_id): + + agreements_client = _get_agreements_client() + agreements, response = agreements_client.getbyconsumer(consumer_id) + return agreements + +def _get_agreement_status(agreement_id): + + agreements_client = _get_agreements_client() + status, response = agreements_client.getstatus(agreement_id) + return status + +def _get_agreement_violations(agreement_id, term=None): + + violations_client = _get_violations_client() + violations, response = violations_client.getbyagreement(agreement_id, term) + return violations + + +class AgreementSimple(APIView): + def build_response(self, code, text): + response = HttpResponse(text, content_type="text/plain", status=code) + return response + + def post( self, request, **kwargs): + #import pdb; pdb.set_trace() + print "------------------------------------------------1" + data = {} + for key, value in request.DATA.items(): + new_key = key + data[new_key] = value + + try: + template_id = data['template_id'] + except: + return self.build_response(400, 'Invalid template_id') + + try: + user = data['user'] + except: + return self.build_response(400, 'Invalid user') + + try: + print "Calling createagreementsimplified with template_id:",template_id,"and user:",user + result = fed4fireservice.createagreementsimplified(template_id, user) + print result + except Exception, e: + print traceback.format_exc() + print '%s (%s)' % (e, type(e)) + + return self.build_response(400, 'Problem creating agreement') + + return self.build_response(200, result) diff --git a/sla/static/css/sla.css b/sla/static/css/sla.css new file mode 100755 index 00000000..0c7702ef --- /dev/null +++ b/sla/static/css/sla.css @@ -0,0 +1,4 @@ +.container{width:100%;} +.left{float:left;width:100px;} +.right{float:right;width:100px;} +.center{margin:0 auto;width:100px;} \ No newline at end of file diff --git a/sla/static/js/sla.js b/sla/static/js/sla.js new file mode 100755 index 00000000..f4bc6124 --- /dev/null +++ b/sla/static/js/sla.js @@ -0,0 +1,167 @@ +/** + * MyPlugin: demonstration plugin + * Version: 0.1 + * Description: Template for writing new plugins and illustrating the different + * possibilities of the plugin API. + * This file is part of the Manifold project + * Requires: js/plugin.js + * URL: http://www.myslice.info + * Author: Jordan Augé + * Copyright: Copyright 2012-2013 UPMC Sorbonne Universités + * License: GPLv3 + */ + +(function($){ + + var MyPlugin = Plugin.extend({ + + /** XXX to check + * @brief Plugin constructor + * @param options : an associative array of setting values + * @param element : + * @return : a jQuery collection of objects on which the plugin is + * applied, which allows to maintain chainability of calls + */ + init: function(options, element) { + // for debugging tools + this.classname="myplugin"; + // Call the parent constructor, see FAQ when forgotten + this._super(options, element); + + /* Member variables */ + + /* Plugin events */ + + /* Setup query and record handlers */ + + // Explain this will allow query events to be handled + // What happens when we don't define some events ? + // Some can be less efficient + this.listen_query(options.query_uuid); + this.listen_query(options.query_uuid, 'all'); + + /* GUI setup and event binding */ + // call function + + }, + + /* PLUGIN EVENTS */ + // on_show like in querytable + + + /* GUI EVENTS */ + + // a function to bind events here: click change + // how to raise manifold events + + + /* GUI MANIPULATION */ + + // We advise you to write function to change behaviour of the GUI + // Will use naming helpers to access content _inside_ the plugin + // always refer to these functions in the remaining of the code + + + + this.id('showEvaluations').click(function() { + alert("WARNING! The experiments are still running. + These SLA evaluations could be different at the end of the experiments." ); + $(".status").css("display",""); + }); + }); + + show_hide_button: function() + { + // this.id, this.el, this.cl, this.elts + // same output as a jquery selector with some guarantees + }, + + /* TEMPLATES */ + + // see in the html template + // How to load a template, use of mustache + + /* QUERY HANDLERS */ + + // How to make sure the plugin is not desynchronized + // He should manifest its interest in filters, fields or records + // functions triggered only if the proper listen is done + + // no prefix + + on_filter_added: function(filter) + { + + }, + + // ... be sure to list all events here + + /* RECORD HANDLERS */ + on_all_new_record: function(record) + { + // + }, + + /* INTERNAL FUNCTIONS */ + _dummy: function() { + // only convention, not strictly enforced at the moment + }, + + }); + + /* Plugin registration */ + $.plugin('MyPlugin', MyPlugin); + + // TODO Here use cases for instanciating plugins in different ways like in the pastie. + +})(jQuery); + + +$(document).ready(function() { + $(".status-success").addClass("icon-ok-sign").attr("title", "Fulfilled") + $(".status-error").addClass("icon-remove-sign").attr("title", "Violated") + $(".status-non-determined").addClass("icon-exclamation-sign").attr("title", "Non determined") + + $(".icon-plus, .icon-minus").click(function(){ $(this).toggleClass("icon-plus icon-minus")}); + console.log("ready") +}); + +$(".agreement_detail").click(function (ev) { // for each edit contact url + ev.preventDefault(); // prevent navigation + var url = $(this).data("form"); // get the contact form url + $("#sla-modal-agreements-{{ a.agreement_id }}").load(url, function () { // load the url into the modal + $(this).modal('show'); // display the modal on url load + }); + return false; // prevent the click propagation +}); + +$('.agreement-detail').live('submit', function () { + $.ajax({ + type: $(this).attr('method'), + url: this.action, + data: $(this).serialize(), + context: this, + success: function (data, status) { + $('#sla-modal-agreements-{{ a.agreement_id }}').html(data); + } + }); + return false; +}); + +$(document).ready(function() { + console.log("consumer_agreements ready"); +}); + +$(".violation-detail").click(function(ev) { // for each edit contact url + ev.preventDefault(); // prevent navigation + var url = $(this).data("href"); + $("#violation-modal").load(url, function() { // load the url into the modal + $(this).modal('show'); // display the modal on url load + }); + return false; // prevent the click propagation +}); + + + this.elts('showEvaluations').click(function(){displayDate()}; + + diff --git a/sla/templates/agreement_detail.html b/sla/templates/agreement_detail.html new file mode 100755 index 00000000..32cdeb5f --- /dev/null +++ b/sla/templates/agreement_detail.html @@ -0,0 +1,77 @@ +

    Agreement detail

    + + \ No newline at end of file diff --git a/sla/templates/consumer_agreements.html b/sla/templates/consumer_agreements.html new file mode 100755 index 00000000..8679bd86 --- /dev/null +++ b/sla/templates/consumer_agreements.html @@ -0,0 +1,26 @@ +
    + +{% for a in agreements %} +
    + {{a.statusclass}} + {{a.context.service}} + - + Detail +
    +
    +
    + {% for tname,t in a.guaranteeterms.items %} +
    + {{t.statusclass}} + {{t.servicelevelobjective.kpiname}} + - + Detail +
    + {% endfor %} +
    +
    +{% empty %} +{% endfor %} +
    + + diff --git a/sla/templates/slice-tab-sla.html b/sla/templates/slice-tab-sla.html new file mode 100755 index 00000000..c605b6d3 --- /dev/null +++ b/sla/templates/slice-tab-sla.html @@ -0,0 +1,140 @@ + +
    +
    + +
    +
    + + + + + + + + + + + + + + + + {% for a in agreements %} + + + + + + + {% if a.guaranteestatus == "VIOLATED" %} + + {% else %} + + {% endif %} + + + + + + + {% for tname,t in a.guaranteeterms.items %} + + + + {% endfor %} + + + {% empty %} + {% endfor %} + + +
    Provider
    iMinds
    {{ a.context.template_id }}{{ a.context.expirationtime }} + {% with a.agreement_id as key %} + {% if enforcements.key == false %} + Disabled + {% else %} + Enabled + {% endif %} + {% endwith %} + + + + View Agreement + {{ t.servicelevelobjective.kpiname }} + {% if t.status == "VIOLATED" %} + + + View Violations + + {% endif %} +
    +
    +
    + + + \ No newline at end of file diff --git a/sla/templates/slice-tab-sla_alternative.html b/sla/templates/slice-tab-sla_alternative.html new file mode 100755 index 00000000..cf19b251 --- /dev/null +++ b/sla/templates/slice-tab-sla_alternative.html @@ -0,0 +1,167 @@ + +
    +
    + + + + + + + + +
    +
    +
    +
    +
    +

    +
    + + Provider +
    +
    {% with agreements|first as a %}{{ a.context.provider }}{% endwith %}
    +

    +
    + + + {% for a in agreements %} + +
    +
    + + + {% if a.guaranteestatus == "VIOLATED" %} + + {% else %} + + {% endif %} + + + + + {% for tname,t in a.guaranteeterms.items %} + + + + {% endfor %} + +
    {{ a.context.template_id }} + + + View Agreement + {{ t.servicelevelobjective.kpiname }} + {% if t.status == "VIOLATED" %} + + + View Violations + + {% endif %} +
    +
    +
    + + + + + {% empty %} + {% endfor %} +
    +
    +
    +
    + + + \ No newline at end of file diff --git a/sla/templates/violations.html b/sla/templates/violations.html new file mode 100755 index 00000000..88c34719 --- /dev/null +++ b/sla/templates/violations.html @@ -0,0 +1,23 @@ +
    + + + + + + + + + {% for v in violations %} + + + + + + {% empty %} + + {% endfor %} +
    #DateActual value
    {{forloop.counter}}{{v.datetime}}{{ v.actual_value|floatformat:"0" }}
    No violations
    +
    +
    +Close +
    diff --git a/sla/templates/violations_template.html b/sla/templates/violations_template.html new file mode 100755 index 00000000..76b4697c --- /dev/null +++ b/sla/templates/violations_template.html @@ -0,0 +1,83 @@ +
    +
    + +
    +

    + Violations +

    + +
    +
    + {# Sanity default: if dd is empty, the values are permutated #} +
    Agreement Id
    +
    {{agreement.agreement_id|default:" "}}
    +
    Service
    +
    {{agreement.context.service_formatted|default:" "}}
    +
    Metric name
    +
    {{guarantee_term.servicelevelobjective.kpiname|default:" "}}
    + {% with guarantee_term.servicelevelobjective.bounds as bounds %} +
    Threshold
    +
    {{bounds.0|default:" "}}
    + {% endwith %} + +
    + +
    + + +
    + +
    + + + + + + + + + {% for v in violations %} + + + + + + {% empty %} + + {% endfor %} +
    #DateActual value
    {{forloop.counter}}{{v.datetime}}{{v.actual_value}}
    No violations
    +
    + +
      + + {% if violations.has_previous %} +
    • <<First
      • +
    • <Previous
      • + {% endif %} + +
    • + + Page {{ violations.number }} of {{ violations.paginator.num_pages }} + +
      • + + {% if violations.has_next %} +
    • Next>
      • +
    • Last>>
      • + {% endif %} + +
    +
    + + \ No newline at end of file diff --git a/sla/templates/violations_template.sublime-workspace b/sla/templates/violations_template.sublime-workspace new file mode 100644 index 00000000..2de266d2 --- /dev/null +++ b/sla/templates/violations_template.sublime-workspace @@ -0,0 +1,191 @@ +{ + "auto_complete": + { + "selected_items": + [ + ] + }, + "buffers": + [ + { + "file": "slice-tab-sla.html", + "settings": + { + "buffer_size": 6021, + "line_ending": "Unix" + } + } + ], + "build_system": "", + "command_palette": + { + "height": 0.0, + "selected_items": + [ + ], + "width": 0.0 + }, + "console": + { + "height": 0.0, + "history": + [ + ] + }, + "distraction_free": + { + "menu_visible": true, + "show_minimap": false, + "show_open_files": false, + "show_tabs": false, + "side_bar_visible": false, + "status_bar_visible": false + }, + "file_history": + [ + ], + "find": + { + "height": 0.0 + }, + "find_in_files": + { + "height": 0.0, + "where_history": + [ + ] + }, + "find_state": + { + "case_sensitive": false, + "find_history": + [ + ], + "highlight": true, + "in_selection": false, + "preserve_case": false, + "regex": false, + "replace_history": + [ + ], + "reverse": false, + "show_context": true, + "use_buffer2": true, + "whole_word": false, + "wrap": true + }, + "groups": + [ + { + "selected": 0, + "sheets": + [ + { + "buffer": 0, + "file": "slice-tab-sla.html", + "semi_transient": false, + "settings": + { + "buffer_size": 6021, + "regions": + { + }, + "selection": + [ + [ + 0, + 0 + ] + ], + "settings": + { + "syntax": "Packages/HTML/HTML.tmLanguage" + }, + "translation.x": 0.0, + "translation.y": 0.0, + "zoom_level": 1.0 + }, + "stack_index": 0, + "type": "text" + } + ] + } + ], + "incremental_find": + { + "height": 0.0 + }, + "input": + { + "height": 0.0 + }, + "layout": + { + "cells": + [ + [ + 0, + 0, + 1, + 1 + ] + ], + "cols": + [ + 0.0, + 1.0 + ], + "rows": + [ + 0.0, + 1.0 + ] + }, + "menu_visible": true, + "output.find_results": + { + "height": 0.0 + }, + "project": "violations_template.sublime-project", + "replace": + { + "height": 0.0 + }, + "save_all_on_build": true, + "select_file": + { + "height": 0.0, + "selected_items": + [ + ], + "width": 0.0 + }, + "select_project": + { + "height": 0.0, + "selected_items": + [ + ], + "width": 0.0 + }, + "select_symbol": + { + "height": 0.0, + "selected_items": + [ + ], + "width": 0.0 + }, + "settings": + { + }, + "show_minimap": true, + "show_open_files": false, + "show_tabs": true, + "side_bar_visible": true, + "side_bar_width": 150.0, + "status_bar_visible": true, + "template_settings": + { + } +} diff --git a/sla/urls.py b/sla/urls.py new file mode 100755 index 00000000..5bc087c8 --- /dev/null +++ b/sla/urls.py @@ -0,0 +1,12 @@ +from django.conf.urls import patterns, url, include + +from sla import slicetabsla + +urlpatterns = patterns('', + url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), + url(r'^(?P[^/]+)/?$', slicetabsla.SLAView.as_view(), name="agreements_summary"), + url(r'^agreements/(?P[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})/detail$', slicetabsla.agreement_details, name='agreement_details'), + url(r'^agreements/(?P[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})/guarantees/(?P\w+)/violations$', slicetabsla.agreement_term_violations, name='agreement_term_violations'), + url(r'^agreements/simplecreate/?$', slicetabsla.AgreementSimple.as_view(), name="agreementsimple"), +) + diff --git a/sla/wsag_helper.py b/sla/wsag_helper.py new file mode 100755 index 00000000..2256ea9a --- /dev/null +++ b/sla/wsag_helper.py @@ -0,0 +1,116 @@ +import re +import datetime + +from slaclient import wsag_model +from slaclient.wsag_model import AgreementStatus +from slaclient.wsag_model import Violation + + +VIOLATED = AgreementStatus.StatusEnum.VIOLATED +NON_DETERMINED = AgreementStatus.StatusEnum.NON_DETERMINED +FULFILLED = AgreementStatus.StatusEnum.FULFILLED + + +def get_violations_bydate(violations): + """Returns a list of violations per date, from a list of violations + + :param violations list[Violation]: + :rtype: list + """ + d = dict() + for v in violations: + assert isinstance(v, Violation) + date = v.datetime.date() + if not date in d: + d[date] = [] + d[date].append(v) + + result = [(key, d[key]) for key in sorted(d.keys(), reverse=True)] + return result + + +class AgreementAnnotator(object): + """Annotates an agreement with the following attributes: + + agreement.guaranteestatus + agreement.statusclass + agreement.guaranteeterms[*].status + agreement.guaranteeterms[*].statusclass + agreement.guaranteeterms[*].nviolations + agreement.guaranteeterms[*].servicelevelobjetive.bounds + + """ + def __init__(self): + pass + + @staticmethod + def _get_statusclass(status): + if status is None or status == "" or status == NON_DETERMINED: + return "non-determined" + return "success" if status == FULFILLED else "error" + + @staticmethod + def _parse_bounds(servicelevel): +# pattern = re.compile(".*BETWEEN *[(]?(.*), *([^)]*)[)]?") + pattern = re.compile(".*GT *([+-]?\\d*\\.\\d+)(?![-+0-9\\.])") + constraint = eval(servicelevel.strip(' \t\n\r')) + m = pattern.match(constraint['constraint']) + return m.groups() + + def _annotate_guaranteeterm(self, term, violations): + # + # Annotate a guarantee term: set bounds and violations + # + level = term.servicelevelobjective.customservicelevel + bounds = AgreementAnnotator._parse_bounds(level) + term.servicelevelobjective.bounds = bounds + + # + # set status attribute if not set before + # + if not hasattr(term, 'status'): + term.status = wsag_model.AgreementStatus.StatusEnum.NON_DETERMINED + # + # TODO: efficiency + # + n = 0 + for violation in violations: + if violation.metric_name == term.servicelevelobjective.kpiname: + n += 1 + term.nviolations = n + + def _annotate_guaranteeterm_by_status( + self, agreement, termstatus, violations): + # + # Annotate a guarantee term: it is different from the previous + # one in that this takes the status into account. + # + name = termstatus.name + status = termstatus.status + + term = agreement.guaranteeterms[name] + term.status = status + term.statusclass = AgreementAnnotator._get_statusclass(status) + self._annotate_guaranteeterm(term, violations) + + def annotate_agreement(self, agreement, status=None, violations=()): + + """Annotate an agreement with certain values needed in the templates + + :param wsag_model.Agreement agreement: agreement to annotate + :param wsag_model.AgreementStatus status: status of the agreement. + :param violations: list of agreement's violations + (wsag_model.Violation[]) + """ + a = agreement + + if status is not None: + a.guaranteestatus = status.guaranteestatus + a.statusclass = self._get_statusclass(status.guaranteestatus) + for termstatus in status.guaranteeterms: + self._annotate_guaranteeterm_by_status( + agreement, termstatus, violations) + else: + a.guaranteestatus = NON_DETERMINED + for termname, term in agreement.guaranteeterms.items(): + self._annotate_guaranteeterm(term, violations) -- 2.43.0