soltesz.py

   1 import os
   2 import sys
   3 import pickle
   4 noserial=False
   5 try:
   6         from PHPSerialize import *
   7         from PHPUnserialize import *
   8 except:
   9         #print >>sys.stderr, "PHPSerial db type not allowed."
  10         noserial=True
  11
  12 import inspect
  13 import shutil
  14 from config2 import config
  15 config = config()
  16
  17 DEBUG= 0
  18 PICKLE_PATH="pdb"
  19
  20 def dbLoad(name, type=None):
  21         return SPickle().load(name, type)
  22
  23 def dbExists(name, type=None):
  24         #if self.config.debug:
  25         #       name = "debug.%s" % name
  26         return SPickle().exists(name, type)
  27
  28 def dbDump(name, obj=None, type=None):
  29         # depth of the dump is 2 now, since we're redirecting to '.dump'
  30         return SPickle().dump(name, obj, type, 2)
  31
  32 def if_cached_else_refresh(cond, refresh, name, function, type=None):
  33         s = SPickle()
  34         if refresh:
  35                 if not config.debug and s.exists("production.%s" % name, type):
  36                         s.remove("production.%s" % name, type)
  37                 if config.debug and s.exists("debug.%s" % name, type):
  38                         s.remove("debug.%s" % name, type)
  39
  40         return if_cached_else(cond, name, function, type)
  41
  42 def if_cached_else(cond, name, function, type=None):
  43         s = SPickle()
  44         if (cond and s.exists("production.%s" % name, type)) or \
  45            (cond and config.debug and s.exists("debug.%s" % name, type)):
  46                 o = s.load(name, type)
  47         else:
  48                 o = function()
  49                 if cond:
  50                         s.dump(name, o, type)   # cache the object using 'name'
  51                         o = s.load(name, type)
  52                 # TODO: what if 'o' hasn't been converted...
  53         return o
  54
  55 class SPickle:
  56         def __init__(self, path=PICKLE_PATH):
  57                 self.path = path
  58
  59         def if_cached_else(self, cond, name, function, type=None):
  60                 if cond and self.exists("production.%s" % name, type):
  61                         o = self.load(name, type)
  62                 else:
  63                         o = function()
  64                         if cond:
  65                                 self.dump(name, o, type)        # cache the object using 'name'
  66                 return o
  67
  68         def __file(self, name, type=None):
  69                 if type == None:
  70                         return "%s/%s.pkl" % (self.path, name)
  71                 else:
  72                         if noserial:
  73                                 raise Exception("No PHPSerializer module available")
  74
  75                         return "%s/%s.phpserial" % (self.path, name)
  76
  77         def exists(self, name, type=None):
  78                 return os.path.exists(self.__file(name, type))
  79
  80         def remove(self, name, type=None):
  81                 return os.remove(self.__file(name, type))
  82
  83         def load(self, name, type=None):
  84                 """
  85                 In debug mode, we should fail if neither file exists.
  86                         if the debug file exists, reset name
  87                         elif the original file exists, make a copy, reset name
  88                         else neither exist, raise an error
  89                 Otherwise, it's normal mode, if the file doesn't exist, raise error
  90                 Load the file
  91                 """
  92
  93                 if config.debug:
  94                         if self.exists("debug.%s" % name, type):
  95                                 name = "debug.%s" % name
  96                         elif self.exists("production.%s" % name, type):
  97                                 debugname = "debug.%s" % name
  98                                 if not self.exists(debugname, type):
  99                                         name = "production.%s" % name
 100                                         shutil.copyfile(self.__file(name, type), self.__file(debugname, type))
 101                                 name = debugname
 102                         else:   # neither exist
 103                                 raise Exception, "No such pickle based on %s" % self.__file("debug.%s" % name, type)
 104                 else:
 105                         if   self.exists("production.%s" % name, type):
 106                                 name = "production.%s" % name
 107                         elif self.exists(name, type):
 108                                 name = name
 109                         else:
 110                                 raise Exception, "No such file %s" % name
 111
 112
 113                 #print "loading %s" % self.__file(name, type)
 114                 f = open(self.__file(name, type), 'r')
 115                 if type == None:
 116                         o = pickle.load(f)
 117                 else:
 118                         if noserial:
 119                                 raise Exception("No PHPSerializer module available")
 120                         s = PHPUnserialize()
 121                         o = s.unserialize(f.read())
 122                 f.close()
 123                 return o
 124
 125
 126         # use the environment to extract the data associated with the local
 127         # variable 'name'
 128         def dump(self, name, obj=None, type=None, depth=1):
 129                 if obj == None:
 130                         o = inspect.getouterframes(inspect.currentframe())
 131                         up1 = o[depth][0] # get the frame one prior to (up from) this frame
 132                         argvals = inspect.getargvalues(up1)
 133                         # TODO: check that 'name' is a local variable; otherwise this would fail.
 134                         obj = argvals[3][name] # extract the local variable name 'name'
 135                 if not os.path.isdir("%s/" % self.path):
 136                         os.mkdir("%s" % self.path)
 137                 if config.debug:
 138                         name = "debug.%s" % name
 139                 else:
 140                         name = "production.%s" % name
 141                 f = open(self.__file(name, type), 'w')
 142                 if type == None:
 143                         pickle.dump(obj, f)
 144                 else:
 145                         if noserial:
 146                                 raise Exception("No PHPSerializer module available")
 147                         s = PHPSerialize()
 148                         f.write(s.serialize(obj))
 149                 f.close()
 150                 return
 151
 152
 153 COMMAND_TIMEOUT = 60
 154 ssh_options = { 'StrictHostKeyChecking':'no',
 155                                 'BatchMode':'yes',
 156                                 'PasswordAuthentication':'no',
 157                                 'ConnectTimeout':'%s' % COMMAND_TIMEOUT}
 158 from select import select
 159 import subprocess
 160 import signal
 161
 162 class Sopen(subprocess.Popen):
 163         def kill(self, signal = signal.SIGTERM):
 164                 os.kill(self.pid, signal)
 165
 166 class CMD:
 167         def __init__(self):
 168                 pass
 169
 170         def run_noexcept(self, cmd, timeout=COMMAND_TIMEOUT*2):
 171
 172                 s = Sopen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
 173                 #(f_in, f_out, f_err) = os.popen3(cmd)
 174                 (f_in, f_out, f_err) = (s.stdin, s.stdout, s.stderr)
 175                 lout, lin, lerr = select([f_out,f_err], [], [], timeout)
 176                 if len(lin) == 0 and len(lout) == 0 and len(lerr) == 0:
 177                         # Reached a timeout!
 178                         #print "TODO: kill subprocess: '%s'" % cmd
 179                         # TODO: kill subprocess??
 180                         s.kill(signal.SIGKILL)
 181                         return ("", "SCRIPTTIMEOUT")
 182                 o_value = f_out.read()
 183                 e_value = ""
 184                 if o_value == "":       # An error has occured
 185                         e_value = f_err.read()
 186
 187                 o_value = o_value.strip()
 188                 e_value = e_value.strip()
 189
 190                 f_out.close()
 191                 f_in.close()
 192                 f_err.close()
 193                 try:
 194                         s.kill()
 195                 except OSError:
 196                         # no such process, due to it already exiting...
 197                         pass
 198
 199                 return (o_value, e_value)
 200
 201         def run_noexcept2(self, cmd):
 202
 203                 (f_in, f_out, f_err) = os.popen3(cmd)
 204                 lout, lin, lerr = select([f_out,f_err], [], [], COMMAND_TIMEOUT*2)
 205                 if len(lin) == 0 and len(lout) == 0 and len(lerr) == 0:
 206                         # Reached a timeout!
 207                         print "TODO: kill subprocess: '%s'" % cmd
 208                         # TODO: kill subprocess??
 209                         return ("", "SCRIPTTIMEOUT")
 210                 o_value = f_out.read()
 211                 e_value = ""
 212                 if o_value == "":       # An error has occured
 213                         e_value = f_err.read()
 214
 215                 o_value = o_value.strip()
 216                 e_value = e_value.strip()
 217
 218                 f_out.close()
 219                 f_in.close()
 220                 f_err.close()
 221                 return (o_value, e_value)
 222
 223         def run(self, cmd):
 224
 225                 (f_in, f_out, f_err) = os.popen3(cmd)
 226                 value = f_out.read()
 227                 if value == "":
 228                         raise Exception, f_err.read()
 229                 value = value.strip()
 230
 231                 f_out.close()
 232                 f_in.close()
 233                 f_err.close()
 234                 return value
 235
 236
 237
 238 class SSH(CMD):
 239         def __init__(self, user, host, options = ssh_options):
 240                 self.options = options
 241                 self.user = user
 242                 self.host = host
 243                 return
 244
 245         def __options_to_str(self):
 246                 options = ""
 247                 for o,v in self.options.iteritems():
 248                         options = options + "-o %s=%s " % (o,v)
 249                 return options
 250
 251         def run(self, cmd):
 252                 cmd = "ssh %s %s@%s '%s'" % (self.__options_to_str(),
 253                                                                         self.user, self.host, cmd)
 254                 return CMD.run(self, cmd)
 255
 256         def get_file(self, rmt_filename, local_filename=None):
 257                 if local_filename == None:
 258                         local_filename = "./"
 259                 cmd = "scp -B %s %s@%s:%s %s" % (self.__options_to_str(),
 260                                                                         self.user, self.host,
 261                                                                         rmt_filename, local_filename)
 262                 # output :
 263                 #       errors will be on stderr,
 264                 #   success will have a blank stderr...
 265                 return CMD.run_noexcept(self, cmd)
 266
 267         def run_noexcept(self, cmd):
 268                 cmd = "ssh %s %s@%s '%s'" % (self.__options_to_str(),
 269                                                                         self.user, self.host, cmd)
 270                 return CMD.run_noexcept(self, cmd)
 271
 272         def runE(self, cmd):
 273                 cmd = "ssh %s %s@%s '%s'" % (self.__options_to_str(),
 274                                                                         self.user, self.host, cmd)
 275                 if ( DEBUG == 1 ):
 276                         print cmd,
 277                 (f_in, f_out, f_err) = os.popen3(cmd)
 278
 279                 value = f_out.read()
 280                 if value == "": # An error has occured
 281                         value = f_err.read()
 282                         value = value.strip()
 283
 284                 if ( DEBUG == 1 ):
 285                         print " == %s" % value
 286                 f_out.close()
 287                 f_in.close()
 288                 f_err.close()
 289                 return value.strip()
 290
 291 import time
 292 class MyTimer:
 293         def __init__(self):
 294                 self.start = time.time()
 295
 296         def end(self):
 297                 self.end = time.time()
 298                 t = self.end-self.start
 299                 return t
 300
 301         def diff(self):
 302                 self.end = time.time()
 303                 t = self.end-self.start
 304                 self.start = self.end
 305                 return t