soltesz.py

   1 import os
   2 import sys
   3 import pickle
   4 noserial=False
   5 try:
   6         from PHPSerialize import *
   7         from PHPUnserialize import *
   8 except:
   9         print >>sys.stderr, "PHPSerial db type not allowed."
  10         noserial=True
  11
  12 import inspect
  13 import shutil
  14 from config import config
  15 config = config()
  16
  17 DEBUG= 0
  18 PICKLE_PATH="pdb"
  19
  20 def dbLoad(name, type=None):
  21         return SPickle().load(name, type)
  22
  23 def dbExists(name, type=None):
  24         #if self.config.debug:
  25         #       name = "debug.%s" % name
  26         return SPickle().exists(name, type)
  27
  28 def dbDump(name, obj=None, type=None):
  29         # depth of the dump is 2 now, since we're redirecting to '.dump'
  30         return SPickle().dump(name, obj, type, 2)
  31
  32 def if_cached_else_refresh(cond, refresh, name, function, type=None):
  33         s = SPickle()
  34         if refresh:
  35                 if not config.debug and s.exists("production.%s" % name, type):
  36                         s.remove("production.%s" % name, type)
  37                 if config.debug and s.exists("debug.%s" % name, type):
  38                         s.remove("debug.%s" % name, type)
  39
  40         return if_cached_else(cond, name, function, type)
  41
  42 def if_cached_else(cond, name, function, type=None):
  43         s = SPickle()
  44         if (cond and s.exists("production.%s" % name, type)) or \
  45            (cond and config.debug and s.exists("debug.%s" % name, type)):
  46                 o = s.load(name, type)
  47         else:
  48                 o = function()
  49                 if cond:
  50                         s.dump(name, o, type)   # cache the object using 'name'
  51                         o = s.load(name, type)
  52                 # TODO: what if 'o' hasn't been converted...
  53         return o
  54
  55 class SPickle:
  56         def __init__(self):
  57                 pass
  58
  59         def if_cached_else(self, cond, name, function, type=None):
  60                 if cond and self.exists("production.%s" % name, type):
  61                         o = self.load(name, type)
  62                 else:
  63                         o = function()
  64                         if cond:
  65                                 self.dump(name, o, type)        # cache the object using 'name'
  66                 return o
  67
  68         def __file(self, name, type=None):
  69                 if type == None:
  70                         return "%s/%s.pkl" % (PICKLE_PATH, name)
  71                 else:
  72                         if noserial:
  73                                 raise Exception("No PHPSerializer module available")
  74
  75                         return "%s/%s.phpserial" % (PICKLE_PATH, name)
  76
  77         def exists(self, name, type=None):
  78                 return os.path.exists(self.__file(name, type))
  79
  80         def remove(self, name, type=None):
  81                 return os.remove(self.__file(name, type))
  82
  83         def load(self, name, type=None):
  84                 """
  85                 In debug mode, we should fail if neither file exists.
  86                         if the debug file exists, reset name
  87                         elif the original file exists, make a copy, reset name
  88                         else neither exist, raise an error
  89                 Otherwise, it's normal mode, if the file doesn't exist, raise error
  90                 Load the file
  91                 """
  92
  93                 if config.debug:
  94                         if self.exists("debug.%s" % name, type):
  95                                 name = "debug.%s" % name
  96                         elif self.exists("production.%s" % name, type):
  97                                 debugname = "debug.%s" % name
  98                                 if not self.exists(debugname, type):
  99                                         name = "production.%s" % name
 100                                         shutil.copyfile(self.__file(name, type), self.__file(debugname, type))
 101                                 name = debugname
 102                         else:   # neither exist
 103                                 raise Exception, "No such pickle based on %s" % self.__file("debug.%s" % name, type)
 104                 else:
 105                         if not self.exists("production.%s" % name, type):
 106                                 raise Exception, "No such file %s" % name
 107                         name = "production.%s" % name
 108
 109                 print "loading %s" % self.__file(name, type)
 110                 f = open(self.__file(name, type), 'r')
 111                 if type == None:
 112                         o = pickle.load(f)
 113                 else:
 114                         if noserial:
 115                                 raise Exception("No PHPSerializer module available")
 116                         s = PHPUnserialize()
 117                         o = s.unserialize(f.read())
 118                 f.close()
 119                 return o
 120
 121
 122         # use the environment to extract the data associated with the local
 123         # variable 'name'
 124         def dump(self, name, obj=None, type=None, depth=1):
 125                 if obj == None:
 126                         o = inspect.getouterframes(inspect.currentframe())
 127                         up1 = o[depth][0] # get the frame one prior to (up from) this frame
 128                         argvals = inspect.getargvalues(up1)
 129                         # TODO: check that 'name' is a local variable; otherwise this would fail.
 130                         obj = argvals[3][name] # extract the local variable name 'name'
 131                 if not os.path.isdir("%s/" % PICKLE_PATH):
 132                         os.mkdir("%s" % PICKLE_PATH)
 133                 if config.debug:
 134                         name = "debug.%s" % name
 135                 else:
 136                         name = "production.%s" % name
 137                 f = open(self.__file(name, type), 'w')
 138                 if type == None:
 139                         pickle.dump(obj, f)
 140                 else:
 141                         if noserial:
 142                                 raise Exception("No PHPSerializer module available")
 143                         s = PHPSerialize()
 144                         f.write(s.serialize(obj))
 145                 f.close()
 146                 return
 147
 148
 149 COMMAND_TIMEOUT = 60
 150 ssh_options = { 'StrictHostKeyChecking':'no',
 151                                 'BatchMode':'yes',
 152                                 'PasswordAuthentication':'no',
 153                                 'ConnectTimeout':'%s' % COMMAND_TIMEOUT}
 154 from select import select
 155 class CMD:
 156         def __init__(self):
 157                 pass
 158
 159         def run_noexcept(self, cmd):
 160
 161                 (f_in, f_out, f_err) = os.popen3(cmd)
 162                 lout, lin, lerr = select([f_out,f_err], [], [], COMMAND_TIMEOUT*2)
 163                 if len(lin) == 0 and len(lout) == 0 and len(lerr) == 0:
 164                         # Reached a timeout!
 165                         print "TODO: kill subprocess: '%s'" % cmd
 166                         # TODO: kill subprocess??
 167                         return ("", "SCRIPTTIMEOUT")
 168                 o_value = f_out.read()
 169                 e_value = ""
 170                 if o_value == "":       # An error has occured
 171                         e_value = f_err.read()
 172
 173                 o_value = o_value.strip()
 174                 e_value = e_value.strip()
 175
 176                 f_out.close()
 177                 f_in.close()
 178                 f_err.close()
 179                 return (o_value, e_value)
 180
 181         def run(self, cmd):
 182
 183                 (f_in, f_out, f_err) = os.popen3(cmd)
 184                 value = f_out.read()
 185                 if value == "":
 186                         raise Exception, f_err.read()
 187                 value = value.strip()
 188
 189                 f_out.close()
 190                 f_in.close()
 191                 f_err.close()
 192                 return value
 193
 194
 195
 196 class SSH(CMD):
 197         def __init__(self, user, host, options = ssh_options):
 198                 self.options = options
 199                 self.user = user
 200                 self.host = host
 201                 return
 202
 203         def __options_to_str(self):
 204                 options = ""
 205                 for o,v in self.options.iteritems():
 206                         options = options + "-o %s=%s " % (o,v)
 207                 return options
 208
 209         def run(self, cmd):
 210                 cmd = "ssh %s %s@%s '%s'" % (self.__options_to_str(),
 211                                                                         self.user, self.host, cmd)
 212                 return CMD.run(self, cmd)
 213
 214         def get_file(self, rmt_filename, local_filename=None):
 215                 if local_filename == None:
 216                         local_filename = "./"
 217                 cmd = "scp -B %s %s@%s:%s %s" % (self.__options_to_str(),
 218                                                                         self.user, self.host,
 219                                                                         rmt_filename, local_filename)
 220                 # output :
 221                 #       errors will be on stderr,
 222                 #   success will have a blank stderr...
 223                 return CMD.run_noexcept(self, cmd)
 224
 225         def run_noexcept(self, cmd):
 226                 cmd = "ssh %s %s@%s '%s'" % (self.__options_to_str(),
 227                                                                         self.user, self.host, cmd)
 228                 return CMD.run_noexcept(self, cmd)
 229
 230         def runE(self, cmd):
 231                 cmd = "ssh %s %s@%s '%s'" % (self.__options_to_str(),
 232                                                                         self.user, self.host, cmd)
 233                 if ( DEBUG == 1 ):
 234                         print cmd,
 235                 (f_in, f_out, f_err) = os.popen3(cmd)
 236
 237                 value = f_out.read()
 238                 if value == "": # An error has occured
 239                         value = f_err.read()
 240                         value = value.strip()
 241
 242                 if ( DEBUG == 1 ):
 243                         print " == %s" % value
 244                 f_out.close()
 245                 f_in.close()
 246                 f_err.close()
 247                 return value.strip()
 248
 249 import time
 250 class MyTimer:
 251         def __init__(self):
 252                 self.start = time.time()
 253
 254         def end(self):
 255                 self.end = time.time()
 256                 t = self.end-self.start
 257                 return t
 258
 259         def diff(self):
 260                 self.end = time.time()
 261                 t = self.end-self.start
 262                 self.start = self.end
 263                 return t