manifold/core/query.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # Query representation
   5 #
   6 # Copyright (C) UPMC Paris Universitas
   7 # Authors:
   8 #   Jordan Augé         <jordan.auge@lip6.fr>
   9 #   Marc-Olivier Buob   <marc-olivier.buob@lip6.fr>
  10 #   Thierry Parmentelat <thierry.parmentelat@inria.fr>
  11
  12 from types                      import StringTypes
  13 from manifold.core.filter         import Filter, Predicate
  14 from manifold.util.frozendict     import frozendict
  15 from manifold.util.type           import returns, accepts
  16 import copy
  17
  18 import json
  19 import uuid
  20
  21 def uniqid ():
  22     return uuid.uuid4().hex
  23
  24 debug=False
  25 debug=True
  26
  27 class ParameterError(StandardError): pass
  28
  29 class Query(object):
  30     """
  31     Implements a TopHat query.
  32
  33     We assume this is a correct DAG specification.
  34
  35     1/ A field designates several tables = OR specification.
  36     2/ The set of fields specifies a AND between OR clauses.
  37     """
  38
  39     #---------------------------------------------------------------------------
  40     # Constructor
  41     #---------------------------------------------------------------------------
  42
  43     def __init__(self, *args, **kwargs):
  44
  45         self.query_uuid = uniqid()
  46
  47         # Initialize optional parameters
  48         self.clear()
  49
  50         #l = len(kwargs.keys())
  51         len_args = len(args)
  52
  53         if len(args) == 1:
  54             if isinstance(args[0], dict):
  55                 kwargs = args[0]
  56                 args = []
  57
  58         # Initialization from a tuple
  59
  60         if len_args in range(2, 7) and type(args) == tuple:
  61             # Note: range(x,y) <=> [x, y[
  62
  63             # XXX UGLY
  64             if len_args == 3:
  65                 self.action = 'get'
  66                 self.params = {}
  67                 self.timestamp     = 'now'
  68                 self.object, self.filters, self.fields = args
  69             elif len_args == 4:
  70                 self.object, self.filters, self.params, self.fields = args
  71                 self.action = 'get'
  72                 self.timestamp     = 'now'
  73             else:
  74                 self.action, self.object, self.filters, self.params, self.fields, self.timestamp = args
  75
  76         # Initialization from a dict
  77         elif "object" in kwargs:
  78             if "action" in kwargs:
  79                 self.action = kwargs["action"]
  80                 del kwargs["action"]
  81             else:
  82                 print "W: defaulting to get action"
  83                 self.action = "get"
  84
  85
  86             self.object = kwargs["object"]
  87             del kwargs["object"]
  88
  89             if "filters" in kwargs:
  90                 self.filters = kwargs["filters"]
  91                 del kwargs["filters"]
  92             else:
  93                 self.filters = Filter([])
  94
  95             if "fields" in kwargs:
  96                 self.fields = set(kwargs["fields"])
  97                 del kwargs["fields"]
  98             else:
  99                 self.fields = set([])
 100
 101             # "update table set x = 3" => params == set
 102             if "params" in kwargs:
 103                 self.params = kwargs["params"]
 104                 del kwargs["params"]
 105             else:
 106                 self.params = {}
 107
 108             if "timestamp" in kwargs:
 109                 self.timestamp = kwargs["timestamp"]
 110                 del kwargs["timestamp"]
 111             else:
 112                 self.timestamp = "now"
 113
 114             if kwargs:
 115                 raise ParameterError, "Invalid parameter(s) : %r" % kwargs.keys()
 116         #else:
 117         #        raise ParameterError, "No valid constructor found for %s : args = %r" % (self.__class__.__name__, args)
 118
 119         if not self.filters: self.filters = Filter([])
 120         if not self.params:  self.params  = {}
 121         if not self.fields:  self.fields  = set([])
 122         if not self.timestamp:      self.timestamp      = "now"
 123
 124         if isinstance(self.filters, list):
 125             f = self.filters
 126             self.filters = Filter([])
 127             for x in f:
 128                 pred = Predicate(x)
 129                 self.filters.add(pred)
 130
 131         if isinstance(self.fields, list):
 132             self.fields = set(self.fields)
 133
 134         for field in self.fields:
 135             if not isinstance(field, StringTypes):
 136                 raise TypeError("Invalid field name %s (string expected, got %s)" % (field, type(field)))
 137
 138     #---------------------------------------------------------------------------
 139     # Helpers
 140     #---------------------------------------------------------------------------
 141
 142     def copy(self):
 143         return copy.deepcopy(self)
 144
 145     def clear(self):
 146         self.action = 'get'
 147         self.object = None
 148         self.filters = Filter([])
 149         self.params  = {}
 150         self.fields  = set([])
 151         self.timestamp  = "now"
 152         self.timestamp  = 'now' # ignored for now
 153
 154     @returns(StringTypes)
 155     def __str__(self):
 156         return "SELECT %(select)s%(from)s%(where)s%(at)s" % {
 157             "select": ", ".join(self.get_select())          if self.get_select()    else "*",
 158             "from"  : "\n  FROM  %s" % self.get_from(),
 159             "where" : "\n  WHERE %s" % self.get_where()     if self.get_where()     else "",
 160             "at"    : "\n  AT    %s" % self.get_timestamp() if self.get_timestamp() else ""
 161         }
 162
 163     @returns(StringTypes)
 164     def __repr__(self):
 165         return "SELECT %s FROM %s WHERE %s" % (
 166             ", ".join(self.get_select()) if self.get_select() else '*',
 167             self.get_from(),
 168             self.get_where()
 169         )
 170
 171     def __key(self):
 172         return (self.action, self.object, self.filters, frozendict(self.params), frozenset(self.fields))
 173
 174     def __hash__(self):
 175         return hash(self.__key())
 176
 177     #---------------------------------------------------------------------------
 178     # Conversion
 179     #---------------------------------------------------------------------------
 180
 181     def to_dict(self):
 182         return {
 183             'action': self.action,
 184             'object': self.object,
 185             'timestamp': self.timestamp,
 186             'filters': self.filters,
 187             'params': self.params,
 188             'fields': list(self.fields)
 189         }
 190
 191     def to_json (self, analyzed_query=None):
 192         query_uuid=self.query_uuid
 193         a=self.action
 194         o=self.object
 195         t=self.timestamp
 196         f=json.dumps (self.filters.to_list())
 197         p=json.dumps (self.params)
 198         c=json.dumps (list(self.fields))
 199         # xxx unique can be removed, but for now we pad the js structure
 200         unique=0
 201
 202         if not analyzed_query:
 203             aq = 'null'
 204         else:
 205             aq = analyzed_query.to_json()
 206         sq="{}"
 207
 208         result= """ new ManifoldQuery('%(a)s', '%(o)s', '%(t)s', %(f)s, %(p)s, %(c)s, %(unique)s, '%(query_uuid)s', %(aq)s, %(sq)s)"""%locals()
 209         if debug: print 'ManifoldQuery.to_json:',result
 210         return result
 211
 212     # this builds a ManifoldQuery object from a dict as received from javascript through its ajax request
 213     # we use a json-encoded string - see manifold.js for the sender part
 214     # e.g. here's what I captured from the server's output
 215     # manifoldproxy.proxy: request.POST <QueryDict: {u'json': [u'{"action":"get","object":"resource","timestamp":"latest","filters":[["slice_hrn","=","ple.inria.omftest"]],"params":[],"fields":["hrn","hostname"],"unique":0,"query_uuid":"436aae70a48141cc826f88e08fbd74b1","analyzed_query":null,"subqueries":{}}']}>
 216     def fill_from_POST (self, POST_dict):
 217         try:
 218             json_string=POST_dict['json']
 219             dict=json.loads(json_string)
 220             for (k,v) in dict.iteritems():
 221                 setattr(self,k,v)
 222         except:
 223             print "Could not decode incoming ajax request as a Query, POST=",POST_dict
 224             if (debug):
 225                 import traceback
 226                 traceback.print_exc()
 227
 228     #---------------------------------------------------------------------------
 229     # Accessors
 230     #---------------------------------------------------------------------------
 231
 232     @returns(StringTypes)
 233     def get_action(self):
 234         return self.action
 235
 236     @returns(frozenset)
 237     def get_select(self):
 238         return frozenset(self.fields)
 239
 240     @returns(StringTypes)
 241     def get_from(self):
 242         return self.object
 243
 244     @returns(Filter)
 245     def get_where(self):
 246         return self.filters
 247
 248     @returns(dict)
 249     def get_params(self):
 250         return self.params
 251
 252     @returns(StringTypes)
 253     def get_timestamp(self):
 254         return self.timestamp
 255
 256 #DEPRECATED#
 257 #DEPRECATED#    def make_filters(self, filters):
 258 #DEPRECATED#        return Filter(filters)
 259 #DEPRECATED#
 260 #DEPRECATED#    def make_fields(self, fields):
 261 #DEPRECATED#        if isinstance(fields, (list, tuple)):
 262 #DEPRECATED#            return set(fields)
 263 #DEPRECATED#        else:
 264 #DEPRECATED#            raise Exception, "Invalid field specification"
 265
 266     #---------------------------------------------------------------------------
 267     # LINQ-like syntax
 268     #---------------------------------------------------------------------------
 269
 270     @classmethod
 271     def action(self, action, object):
 272         query = Query()
 273         query.action = action
 274         query.object = object
 275         return query
 276
 277     @classmethod
 278     def get(self, object): return self.action('get', object)
 279
 280     @classmethod
 281     def update(self, object): return self.action('update', object)
 282
 283     @classmethod
 284     def create(self, object): return self.action('create', object)
 285
 286     @classmethod
 287     def delete(self, object): return self.action('delete', object)
 288
 289     @classmethod
 290     def execute(self, object): return self.action('execute', object)
 291
 292     def at(self, timestamp):
 293         self.timestamp = timestamp
 294         return self
 295
 296     def filter_by(self, *args):
 297         if len(args) == 1:
 298             filters = args[0]
 299             if not isinstance(filters, (set, list, tuple, Filter)):
 300                 filters = [filters]
 301             for predicate in filters:
 302                 self.filters.add(predicate)
 303         elif len(args) == 3:
 304             predicate = Predicate(*args)
 305             self.filters.add(predicate)
 306         else:
 307             raise Exception, 'Invalid expression for filter'
 308         return self
 309
 310     def select(self, fields=None):
 311         if not fields:
 312             # Delete all fields
 313             self.fields = set()
 314             return self
 315         if not isinstance(fields, (set, list, tuple)):
 316             fields = [fields]
 317         for field in fields:
 318             self.fields.add(field)
 319         return self
 320
 321     def set(self, params):
 322         self.params.update(params)
 323         return self
 324
 325 class AnalyzedQuery(Query):
 326
 327     # XXX we might need to propagate special parameters sur as DEBUG, etc.
 328
 329     def __init__(self, query=None, metadata=None):
 330         self.clear()
 331         self.metadata = metadata
 332         if query:
 333             self.query_uuid = query.query_uuid
 334             self.analyze(query)
 335         else:
 336             self.query_uuid = uniqid()
 337
 338     @returns(StringTypes)
 339     def __str__(self):
 340         out = []
 341         fields = self.get_select()
 342         fields = ", ".join(fields) if fields else '*'
 343         out.append("SELECT %s FROM %s WHERE %s" % (
 344             fields,
 345             self.get_from(),
 346             self.get_where()
 347         ))
 348         cpt = 1
 349         for method, subquery in self.subqueries():
 350             out.append('  [SQ #%d : %s] %s' % (cpt, method, str(subquery)))
 351             cpt += 1
 352
 353         return "\n".join(out)
 354
 355     def clear(self):
 356         super(AnalyzedQuery, self).clear()
 357         self._subqueries = {}
 358
 359     def subquery(self, method):
 360         # Allows for the construction of a subquery
 361         if not method in self._subqueries:
 362             analyzed_query = AnalyzedQuery(metadata=self.metadata)
 363             analyzed_query.action = self.action
 364             if self.metadata:
 365                 try:
 366                     type = self.metadata.get_field_type(self.object, method)
 367                 except ValueError ,e: # backwards 1..N
 368                     type = method
 369             else:
 370                 type = method
 371             analyzed_query.object = type
 372             self._subqueries[method] = analyzed_query
 373         return self._subqueries[method]
 374
 375     def get_subquery(self, method):
 376         return self._subqueries.get(method, None)
 377
 378     def remove_subquery(self, method):
 379         del self._subqueries[method]
 380
 381     def get_subquery_names(self):
 382         return set(self._subqueries.keys())
 383
 384     def subqueries(self):
 385         for method, subquery in self._subqueries.iteritems():
 386             yield (method, subquery)
 387
 388     def filter_by(self, filters):
 389         if not filters: return self
 390         if not isinstance(filters, (set, list, tuple, Filter)):
 391             filters = [filters]
 392         for predicate in filters:
 393             if '.' in predicate.key:
 394                 method, subkey = pred.key.split('.', 1)
 395                 # Method contains the name of the subquery, we need the type
 396                 # XXX type = self.metadata.get_field_type(self.object, method)
 397                 sub_pred = Predicate(subkey, pred.op, pred.value)
 398                 self.subquery(method).filter_by(sub_pred)
 399             else:
 400                 super(AnalyzedQuery, self).filter_by(predicate)
 401         return self
 402
 403     def select(self, fields):
 404         if not isinstance(fields, (set, list, tuple)):
 405             fields = [fields]
 406         for field in fields:
 407             if '.' in field:
 408                 method, subfield = field.split('.', 1)
 409                 # Method contains the name of the subquery, we need the type
 410                 # XXX type = self.metadata.get_field_type(self.object, method)
 411                 self.subquery(method).select(subfield)
 412             else:
 413                 super(AnalyzedQuery, self).select(field)
 414         return self
 415
 416     def set(self, params):
 417         for param, value in self.params.items():
 418             if '.' in param:
 419                 method, subparam = param.split('.', 1)
 420                 # Method contains the name of the subquery, we need the type
 421                 # XXX type = self.metadata.get_field_type(self.object, method)
 422                 self.subquery(method).set({subparam: value})
 423             else:
 424                 super(AnalyzedQuery, self).set({param: value})
 425         return self
 426
 427     def analyze(self, query):
 428         self.clear()
 429         self.action = query.action
 430         self.object = query.object
 431         self.filter_by(query.filters)
 432         self.set(query.params)
 433         self.select(query.fields)
 434
 435     def to_json (self):
 436         query_uuid=self.query_uuid
 437         a=self.action
 438         o=self.object
 439         t=self.timestamp
 440         f=json.dumps (self.filters.to_list())
 441         p=json.dumps (self.params)
 442         c=json.dumps (list(self.fields))
 443         # xxx unique can be removed, but for now we pad the js structure
 444         unique=0
 445
 446         aq = 'null'
 447         sq=", ".join ( [ "'%s':%s" % (object, subquery.to_json())
 448                   for (object, subquery) in self._subqueries.iteritems()])
 449         sq="{%s}"%sq
 450
 451         result= """ new ManifoldQuery('%(a)s', '%(o)s', '%(t)s', %(f)s, %(p)s, %(c)s, %(unique)s, '%(query_uuid)s', %(aq)s, %(sq)s)"""%locals()
 452         if debug: print 'ManifoldQuery.to_json:',result
 453         return result