soltesz.py

   1 import os
   2 import sys
   3 import pickle
   4 noserial=False
   5 try:
   6         from PHPSerialize import *
   7         from PHPUnserialize import *
   8 except:
   9         #print >>sys.stderr, "PHPSerial db type not allowed."
  10         noserial=True
  11
  12 import inspect
  13 import shutil
  14 from config2 import config
  15 config = config()
  16
  17 DEBUG= 0
  18 PICKLE_PATH="pdb"
  19
  20 class ExceptionTimeout(Exception): pass
  21
  22 def dbLoad(name, type=None):
  23         return SPickle().load(name, type)
  24
  25 def dbExists(name, type=None):
  26         #if self.config.debug:
  27         #       name = "debug.%s" % name
  28         return SPickle().exists(name, type)
  29
  30 def dbDump(name, obj=None, type=None):
  31         # depth of the dump is 2 now, since we're redirecting to '.dump'
  32         return SPickle().dump(name, obj, type, 2)
  33
  34 def if_cached_else_refresh(cond, refresh, name, function, type=None):
  35         s = SPickle()
  36         if refresh:
  37                 if not config.debug and s.exists("production.%s" % name, type):
  38                         s.remove("production.%s" % name, type)
  39                 if config.debug and s.exists("debug.%s" % name, type):
  40                         s.remove("debug.%s" % name, type)
  41
  42         return if_cached_else(cond, name, function, type)
  43
  44 def if_cached_else(cond, name, function, type=None):
  45         s = SPickle()
  46         if (cond and s.exists("production.%s" % name, type)) or \
  47            (cond and config.debug and s.exists("debug.%s" % name, type)):
  48                 o = s.load(name, type)
  49         else:
  50                 o = function()
  51                 if cond:
  52                         s.dump(name, o, type)   # cache the object using 'name'
  53                         o = s.load(name, type)
  54                 # TODO: what if 'o' hasn't been converted...
  55         return o
  56
  57 class SPickle:
  58         def __init__(self, path=PICKLE_PATH):
  59                 self.path = path
  60
  61         def if_cached_else(self, cond, name, function, type=None):
  62                 if cond and self.exists("production.%s" % name, type):
  63                         o = self.load(name, type)
  64                 else:
  65                         o = function()
  66                         if cond:
  67                                 self.dump(name, o, type)        # cache the object using 'name'
  68                 return o
  69
  70         def __file(self, name, type=None):
  71                 if type == None:
  72                         return "%s/%s.pkl" % (self.path, name)
  73                 else:
  74                         if noserial:
  75                                 raise Exception("No PHPSerializer module available")
  76
  77                         return "%s/%s.phpserial" % (self.path, name)
  78
  79         def exists(self, name, type=None):
  80                 return os.path.exists(self.__file(name, type))
  81
  82         def remove(self, name, type=None):
  83                 return os.remove(self.__file(name, type))
  84
  85         def load(self, name, type=None):
  86                 """
  87                 In debug mode, we should fail if neither file exists.
  88                         if the debug file exists, reset name
  89                         elif the original file exists, make a copy, reset name
  90                         else neither exist, raise an error
  91                 Otherwise, it's normal mode, if the file doesn't exist, raise error
  92                 Load the file
  93                 """
  94
  95                 if config.debug:
  96                         if self.exists("debug.%s" % name, type):
  97                                 name = "debug.%s" % name
  98                         elif self.exists("production.%s" % name, type):
  99                                 debugname = "debug.%s" % name
 100                                 if not self.exists(debugname, type):
 101                                         name = "production.%s" % name
 102                                         shutil.copyfile(self.__file(name, type), self.__file(debugname, type))
 103                                 name = debugname
 104                         else:   # neither exist
 105                                 raise Exception, "No such pickle based on %s" % self.__file("debug.%s" % name, type)
 106                 else:
 107                         if   self.exists("production.%s" % name, type):
 108                                 name = "production.%s" % name
 109                         elif self.exists(name, type):
 110                                 name = name
 111                         else:
 112                                 raise Exception, "No such file %s" % name
 113
 114
 115                 #print "loading %s" % self.__file(name, type)
 116                 f = open(self.__file(name, type), 'r')
 117                 if type == None:
 118                         o = pickle.load(f)
 119                 else:
 120                         if noserial:
 121                                 raise Exception("No PHPSerializer module available")
 122                         s = PHPUnserialize()
 123                         o = s.unserialize(f.read())
 124                 f.close()
 125                 return o
 126
 127
 128         # use the environment to extract the data associated with the local
 129         # variable 'name'
 130         def dump(self, name, obj=None, type=None, depth=1):
 131                 if obj == None:
 132                         o = inspect.getouterframes(inspect.currentframe())
 133                         up1 = o[depth][0] # get the frame one prior to (up from) this frame
 134                         argvals = inspect.getargvalues(up1)
 135                         # TODO: check that 'name' is a local variable; otherwise this would fail.
 136                         obj = argvals[3][name] # extract the local variable name 'name'
 137                 if not os.path.isdir("%s/" % self.path):
 138                         os.mkdir("%s" % self.path)
 139                 if config.debug:
 140                         name = "debug.%s" % name
 141                 else:
 142                         name = "production.%s" % name
 143                 f = open(self.__file(name, type), 'w')
 144                 if type == None:
 145                         pickle.dump(obj, f)
 146                 else:
 147                         if noserial:
 148                                 raise Exception("No PHPSerializer module available")
 149                         s = PHPSerialize()
 150                         f.write(s.serialize(obj))
 151                 f.close()
 152                 return
 153
 154
 155 COMMAND_TIMEOUT = 60
 156 ssh_options = { 'StrictHostKeyChecking':'no',
 157                                 'BatchMode':'yes',
 158                                 'PasswordAuthentication':'no',
 159                                 'ConnectTimeout':'%s' % COMMAND_TIMEOUT}
 160 from select import select
 161 import subprocess
 162 import signal
 163
 164 class Sopen(subprocess.Popen):
 165         def kill(self, signal = signal.SIGTERM):
 166                 os.kill(self.pid, signal)
 167
 168 class CMD:
 169         def __init__(self):
 170                 pass
 171
 172         def run_noexcept(self, cmd, timeout=COMMAND_TIMEOUT*2):
 173
 174                 try:
 175                         return CMD.run(self,cmd,timeout)
 176                 except ExceptionTimeout:
 177                         import traceback; print traceback.print_exc()
 178                         return ("", "SCRIPTTIMEOUT")
 179
 180
 181 #               s = Sopen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
 182 #               #(f_in, f_out, f_err) = os.popen3(cmd)
 183 #               (f_in, f_out, f_err) = (s.stdin, s.stdout, s.stderr)
 184 #               lout, lin, lerr = select([f_out,f_err], [], [], timeout)
 185 #               if len(lin) == 0 and len(lout) == 0 and len(lerr) == 0:
 186 #                       # Reached a timeout!  Nuke process so it does not hang.
 187 #                       s.kill(signal.SIGKILL)
 188 #                       return ("", "SCRIPTTIMEOUT")
 189 #               o_value = f_out.read()
 190 #               e_value = ""
 191 #               if o_value == "":       # An error has occured
 192 #                       e_value = f_err.read()
 193 #
 194 #               o_value = o_value.strip()
 195 #               e_value = e_value.strip()
 196 #
 197 #               f_out.close()
 198 #               f_in.close()
 199 #               f_err.close()
 200 #               try:
 201 #                       s.kill()
 202 #               except OSError:
 203 #                       # no such process, due to it already exiting...
 204 #                       pass
 205 #
 206 #               return (o_value, e_value)
 207         def system(self, cmd, timeout=COMMAND_TIMEOUT*2):
 208                 (o,e) = self.run(cmd, timeout)
 209                 self.output = o
 210                 self.error = e
 211                 if self.s.returncode is None:
 212                         self.s.wait()
 213                 return self.s.returncode
 214
 215         def run(self, cmd, timeout=COMMAND_TIMEOUT*2):
 216
 217                 s = Sopen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
 218                 self.s = s
 219                 (f_in, f_out, f_err) = (s.stdin, s.stdout, s.stderr)
 220                 lout, lin, lerr = select([f_out,f_err], [], [], timeout)
 221                 if len(lin) == 0 and len(lout) == 0 and len(lerr) == 0:
 222                         # Reached a timeout!  Nuke process so it does not hang.
 223                         s.kill(signal.SIGKILL)
 224                         raise ExceptionTimeout("TIMEOUT Running: %s" % cmd)
 225                 o_value = f_out.read()
 226                 e_value = ""
 227                 if o_value == "":       # An error has occured
 228                         e_value = f_err.read()
 229
 230                 o_value = o_value.strip()
 231                 e_value = e_value.strip()
 232
 233                 f_out.close()
 234                 f_in.close()
 235                 f_err.close()
 236                 try:
 237                         s.kill()
 238                 except OSError:
 239                         # no such process, due to it already exiting...
 240                         pass
 241
 242                 return (o_value, e_value)
 243
 244
 245 class SSH(CMD):
 246         def __init__(self, user, host, options = ssh_options):
 247                 self.options = options
 248                 self.user = user
 249                 self.host = host
 250                 return
 251
 252         def __options_to_str(self):
 253                 options = ""
 254                 for o,v in self.options.iteritems():
 255                         options = options + "-o %s=%s " % (o,v)
 256                 return options
 257
 258         def run(self, cmd, timeout=COMMAND_TIMEOUT*2):
 259                 cmd = "ssh %s %s@%s '%s'" % (self.__options_to_str(),
 260                                                                         self.user, self.host, cmd)
 261                 return CMD.run(self, cmd, timeout)
 262
 263         def get_file(self, rmt_filename, local_filename=None):
 264                 if local_filename == None:
 265                         local_filename = "./"
 266                 cmd = "scp -B %s %s@%s:%s %s" % (self.__options_to_str(),
 267                                                                         self.user, self.host,
 268                                                                         rmt_filename, local_filename)
 269                 # output :
 270                 #       errors will be on stderr,
 271                 #   success will have a blank stderr...
 272                 return CMD.run_noexcept(self, cmd)
 273
 274         def run_noexcept(self, cmd):
 275                 cmd = "ssh %s %s@%s '%s'" % (self.__options_to_str(),
 276                                                                         self.user, self.host, cmd)
 277                 return CMD.run_noexcept(self, cmd)
 278
 279         def runE(self, cmd):
 280                 cmd = "ssh %s %s@%s '%s'" % (self.__options_to_str(),
 281                                                                         self.user, self.host, cmd)
 282                 if ( DEBUG == 1 ):
 283                         print cmd,
 284                 (f_in, f_out, f_err) = os.popen3(cmd)
 285
 286                 value = f_out.read()
 287                 if value == "": # An error has occured
 288                         value = f_err.read()
 289                         value = value.strip()
 290
 291                 if ( DEBUG == 1 ):
 292                         print " == %s" % value
 293                 f_out.close()
 294                 f_in.close()
 295                 f_err.close()
 296                 return value.strip()
 297
 298 import time
 299 class MyTimer:
 300         def __init__(self):
 301                 self.start = time.time()
 302
 303         def end(self):
 304                 self.end = time.time()
 305                 t = self.end-self.start
 306                 return t
 307
 308         def diff(self):
 309                 self.end = time.time()
 310                 t = self.end-self.start
 311                 self.start = self.end
 312                 return t