soltesz.py

   1 import os
   2 import sys
   3 import pickle
   4 noserial=False
   5 try:
   6         from PHPSerialize import *
   7         from PHPUnserialize import *
   8 except:
   9         #print >>sys.stderr, "PHPSerial db type not allowed."
  10         noserial=True
  11
  12 import inspect
  13 import shutil
  14 from config2 import config
  15 config = config()
  16
  17 DEBUG= 0
  18 PICKLE_PATH="pdb"
  19
  20 def dbLoad(name, type=None):
  21         return SPickle().load(name, type)
  22
  23 def dbExists(name, type=None):
  24         #if self.config.debug:
  25         #       name = "debug.%s" % name
  26         return SPickle().exists(name, type)
  27
  28 def dbDump(name, obj=None, type=None):
  29         # depth of the dump is 2 now, since we're redirecting to '.dump'
  30         return SPickle().dump(name, obj, type, 2)
  31
  32 def if_cached_else_refresh(cond, refresh, name, function, type=None):
  33         s = SPickle()
  34         if refresh:
  35                 if not config.debug and s.exists("production.%s" % name, type):
  36                         s.remove("production.%s" % name, type)
  37                 if config.debug and s.exists("debug.%s" % name, type):
  38                         s.remove("debug.%s" % name, type)
  39
  40         return if_cached_else(cond, name, function, type)
  41
  42 def if_cached_else(cond, name, function, type=None):
  43         s = SPickle()
  44         if (cond and s.exists("production.%s" % name, type)) or \
  45            (cond and config.debug and s.exists("debug.%s" % name, type)):
  46                 o = s.load(name, type)
  47         else:
  48                 o = function()
  49                 if cond:
  50                         s.dump(name, o, type)   # cache the object using 'name'
  51                         o = s.load(name, type)
  52                 # TODO: what if 'o' hasn't been converted...
  53         return o
  54
  55 class SPickle:
  56         def __init__(self, path=PICKLE_PATH):
  57                 self.path = path
  58
  59         def if_cached_else(self, cond, name, function, type=None):
  60                 if cond and self.exists("production.%s" % name, type):
  61                         o = self.load(name, type)
  62                 else:
  63                         o = function()
  64                         if cond:
  65                                 self.dump(name, o, type)        # cache the object using 'name'
  66                 return o
  67
  68         def __file(self, name, type=None):
  69                 if type == None:
  70                         return "%s/%s.pkl" % (self.path, name)
  71                 else:
  72                         if noserial:
  73                                 raise Exception("No PHPSerializer module available")
  74
  75                         return "%s/%s.phpserial" % (self.path, name)
  76
  77         def exists(self, name, type=None):
  78                 return os.path.exists(self.__file(name, type))
  79
  80         def remove(self, name, type=None):
  81                 return os.remove(self.__file(name, type))
  82
  83         def load(self, name, type=None):
  84                 """
  85                 In debug mode, we should fail if neither file exists.
  86                         if the debug file exists, reset name
  87                         elif the original file exists, make a copy, reset name
  88                         else neither exist, raise an error
  89                 Otherwise, it's normal mode, if the file doesn't exist, raise error
  90                 Load the file
  91                 """
  92
  93                 if config.debug:
  94                         if self.exists("debug.%s" % name, type):
  95                                 name = "debug.%s" % name
  96                         elif self.exists("production.%s" % name, type):
  97                                 debugname = "debug.%s" % name
  98                                 if not self.exists(debugname, type):
  99                                         name = "production.%s" % name
 100                                         shutil.copyfile(self.__file(name, type), self.__file(debugname, type))
 101                                 name = debugname
 102                         else:   # neither exist
 103                                 raise Exception, "No such pickle based on %s" % self.__file("debug.%s" % name, type)
 104                 else:
 105                         if   self.exists("production.%s" % name, type):
 106                                 name = "production.%s" % name
 107                         elif self.exists(name, type):
 108                                 name = name
 109                         else:
 110                                 raise Exception, "No such file %s" % name
 111
 112
 113                 #print "loading %s" % self.__file(name, type)
 114                 f = open(self.__file(name, type), 'r')
 115                 if type == None:
 116                         o = pickle.load(f)
 117                 else:
 118                         if noserial:
 119                                 raise Exception("No PHPSerializer module available")
 120                         s = PHPUnserialize()
 121                         o = s.unserialize(f.read())
 122                 f.close()
 123                 return o
 124
 125
 126         # use the environment to extract the data associated with the local
 127         # variable 'name'
 128         def dump(self, name, obj=None, type=None, depth=1):
 129                 if obj == None:
 130                         o = inspect.getouterframes(inspect.currentframe())
 131                         up1 = o[depth][0] # get the frame one prior to (up from) this frame
 132                         argvals = inspect.getargvalues(up1)
 133                         # TODO: check that 'name' is a local variable; otherwise this would fail.
 134                         obj = argvals[3][name] # extract the local variable name 'name'
 135                 if not os.path.isdir("%s/" % self.path):
 136                         os.mkdir("%s" % self.path)
 137                 if config.debug:
 138                         name = "debug.%s" % name
 139                 else:
 140                         name = "production.%s" % name
 141                 f = open(self.__file(name, type), 'w')
 142                 if type == None:
 143                         pickle.dump(obj, f)
 144                 else:
 145                         if noserial:
 146                                 raise Exception("No PHPSerializer module available")
 147                         s = PHPSerialize()
 148                         f.write(s.serialize(obj))
 149                 f.close()
 150                 return
 151
 152
 153 COMMAND_TIMEOUT = 60
 154 ssh_options = { 'StrictHostKeyChecking':'no',
 155                                 'BatchMode':'yes',
 156                                 'PasswordAuthentication':'no',
 157                                 'ConnectTimeout':'%s' % COMMAND_TIMEOUT}
 158 from select import select
 159 import subprocess
 160 import signal
 161
 162 class Sopen(subprocess.Popen):
 163         def kill(self, signal = signal.SIGTERM):
 164                 os.kill(self.pid, signal)
 165
 166 class CMD:
 167         def __init__(self):
 168                 pass
 169
 170         def run_noexcept(self, cmd):
 171
 172                 s = Sopen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
 173                 #(f_in, f_out, f_err) = os.popen3(cmd)
 174                 (f_in, f_out, f_err) = (s.stdin, s.stdout, s.stderr)
 175                 lout, lin, lerr = select([f_out,f_err], [], [], COMMAND_TIMEOUT*2)
 176                 if len(lin) == 0 and len(lout) == 0 and len(lerr) == 0:
 177                         # Reached a timeout!
 178                         #print "TODO: kill subprocess: '%s'" % cmd
 179                         # TODO: kill subprocess??
 180                         s.kill()
 181                         return ("", "SCRIPTTIMEOUT")
 182                 o_value = f_out.read()
 183                 e_value = ""
 184                 if o_value == "":       # An error has occured
 185                         e_value = f_err.read()
 186
 187                 o_value = o_value.strip()
 188                 e_value = e_value.strip()
 189
 190                 f_out.close()
 191                 f_in.close()
 192                 f_err.close()
 193                 return (o_value, e_value)
 194
 195         def run_noexcept2(self, cmd):
 196
 197                 (f_in, f_out, f_err) = os.popen3(cmd)
 198                 lout, lin, lerr = select([f_out,f_err], [], [], COMMAND_TIMEOUT*2)
 199                 if len(lin) == 0 and len(lout) == 0 and len(lerr) == 0:
 200                         # Reached a timeout!
 201                         print "TODO: kill subprocess: '%s'" % cmd
 202                         # TODO: kill subprocess??
 203                         return ("", "SCRIPTTIMEOUT")
 204                 o_value = f_out.read()
 205                 e_value = ""
 206                 if o_value == "":       # An error has occured
 207                         e_value = f_err.read()
 208
 209                 o_value = o_value.strip()
 210                 e_value = e_value.strip()
 211
 212                 f_out.close()
 213                 f_in.close()
 214                 f_err.close()
 215                 return (o_value, e_value)
 216
 217         def run(self, cmd):
 218
 219                 (f_in, f_out, f_err) = os.popen3(cmd)
 220                 value = f_out.read()
 221                 if value == "":
 222                         raise Exception, f_err.read()
 223                 value = value.strip()
 224
 225                 f_out.close()
 226                 f_in.close()
 227                 f_err.close()
 228                 return value
 229
 230
 231
 232 class SSH(CMD):
 233         def __init__(self, user, host, options = ssh_options):
 234                 self.options = options
 235                 self.user = user
 236                 self.host = host
 237                 return
 238
 239         def __options_to_str(self):
 240                 options = ""
 241                 for o,v in self.options.iteritems():
 242                         options = options + "-o %s=%s " % (o,v)
 243                 return options
 244
 245         def run(self, cmd):
 246                 cmd = "ssh %s %s@%s '%s'" % (self.__options_to_str(),
 247                                                                         self.user, self.host, cmd)
 248                 return CMD.run(self, cmd)
 249
 250         def get_file(self, rmt_filename, local_filename=None):
 251                 if local_filename == None:
 252                         local_filename = "./"
 253                 cmd = "scp -B %s %s@%s:%s %s" % (self.__options_to_str(),
 254                                                                         self.user, self.host,
 255                                                                         rmt_filename, local_filename)
 256                 # output :
 257                 #       errors will be on stderr,
 258                 #   success will have a blank stderr...
 259                 return CMD.run_noexcept(self, cmd)
 260
 261         def run_noexcept(self, cmd):
 262                 cmd = "ssh %s %s@%s '%s'" % (self.__options_to_str(),
 263                                                                         self.user, self.host, cmd)
 264                 return CMD.run_noexcept(self, cmd)
 265
 266         def runE(self, cmd):
 267                 cmd = "ssh %s %s@%s '%s'" % (self.__options_to_str(),
 268                                                                         self.user, self.host, cmd)
 269                 if ( DEBUG == 1 ):
 270                         print cmd,
 271                 (f_in, f_out, f_err) = os.popen3(cmd)
 272
 273                 value = f_out.read()
 274                 if value == "": # An error has occured
 275                         value = f_err.read()
 276                         value = value.strip()
 277
 278                 if ( DEBUG == 1 ):
 279                         print " == %s" % value
 280                 f_out.close()
 281                 f_in.close()
 282                 f_err.close()
 283                 return value.strip()
 284
 285 import time
 286 class MyTimer:
 287         def __init__(self):
 288                 self.start = time.time()
 289
 290         def end(self):
 291                 self.end = time.time()
 292                 t = self.end-self.start
 293                 return t
 294
 295         def diff(self):
 296                 self.end = time.time()
 297                 t = self.end-self.start
 298                 self.start = self.end
 299                 return t