soltesz.py

   1 import os
   2 import sys
   3 import pickle
   4 noserial=False
   5 try:
   6         from PHPSerialize import *
   7         from PHPUnserialize import *
   8 except:
   9         #print >>sys.stderr, "PHPSerial db type not allowed."
  10         noserial=True
  11
  12 import inspect
  13 import shutil
  14 from config import config as cfg
  15 config = cfg()
  16
  17 import monitorconfig
  18
  19 DEBUG= 0
  20 PICKLE_PATH=monitorconfig.MONITOR_DATA_ROOT
  21
  22 class ExceptionTimeout(Exception): pass
  23
  24 def dbLoad(name, type=None):
  25         return SPickle().load(name, type)
  26
  27 def dbExists(name, type=None):
  28         #if self.config.debug:
  29         #       name = "debug.%s" % name
  30         return SPickle().exists(name, type)
  31
  32 def dbDump(name, obj=None, type=None):
  33         # depth of the dump is 2 now, since we're redirecting to '.dump'
  34         return SPickle().dump(name, obj, type, 2)
  35
  36 def if_cached_else_refresh(cond, refresh, name, function, type=None):
  37         s = SPickle()
  38         if refresh:
  39                 if not config.debug and s.exists("production.%s" % name, type):
  40                         s.remove("production.%s" % name, type)
  41                 if config.debug and s.exists("debug.%s" % name, type):
  42                         s.remove("debug.%s" % name, type)
  43
  44         return if_cached_else(cond, name, function, type)
  45
  46 def if_cached_else(cond, name, function, type=None):
  47         s = SPickle()
  48         if (cond and s.exists("production.%s" % name, type)) or \
  49            (cond and config.debug and s.exists("debug.%s" % name, type)):
  50                 o = s.load(name, type)
  51         else:
  52                 o = function()
  53                 if cond:
  54                         s.dump(name, o, type)   # cache the object using 'name'
  55                         o = s.load(name, type)
  56                 # TODO: what if 'o' hasn't been converted...
  57         return o
  58
  59 class SPickle:
  60         def __init__(self, path=PICKLE_PATH):
  61                 self.path = path
  62
  63         def if_cached_else(self, cond, name, function, type=None):
  64                 if cond and self.exists("production.%s" % name, type):
  65                         o = self.load(name, type)
  66                 else:
  67                         o = function()
  68                         if cond:
  69                                 self.dump(name, o, type)        # cache the object using 'name'
  70                 return o
  71
  72         def __file(self, name, type=None):
  73                 if type == None:
  74                         return "%s/%s.pkl" % (self.path, name)
  75                 else:
  76                         if noserial:
  77                                 raise Exception("No PHPSerializer module available")
  78
  79                         return "%s/%s.phpserial" % (self.path, name)
  80
  81         def exists(self, name, type=None):
  82                 return os.path.exists(self.__file(name, type))
  83
  84         def remove(self, name, type=None):
  85                 return os.remove(self.__file(name, type))
  86
  87         def load(self, name, type=None):
  88                 """
  89                 In debug mode, we should fail if neither file exists.
  90                         if the debug file exists, reset name
  91                         elif the original file exists, make a copy, reset name
  92                         else neither exist, raise an error
  93                 Otherwise, it's normal mode, if the file doesn't exist, raise error
  94                 Load the file
  95                 """
  96
  97                 if config.debug:
  98                         if self.exists("debug.%s" % name, type):
  99                                 name = "debug.%s" % name
 100                         elif self.exists("production.%s" % name, type):
 101                                 debugname = "debug.%s" % name
 102                                 if not self.exists(debugname, type):
 103                                         name = "production.%s" % name
 104                                         shutil.copyfile(self.__file(name, type), self.__file(debugname, type))
 105                                 name = debugname
 106                         else:   # neither exist
 107                                 raise Exception, "No such pickle based on %s" % self.__file("debug.%s" % name, type)
 108                 else:
 109                         if   self.exists("production.%s" % name, type):
 110                                 name = "production.%s" % name
 111                         elif self.exists(name, type):
 112                                 name = name
 113                         else:
 114                                 raise Exception, "No such file %s" % name
 115
 116
 117                 #print "loading %s" % self.__file(name, type)
 118                 f = open(self.__file(name, type), 'r')
 119                 if type == None:
 120                         o = pickle.load(f)
 121                 else:
 122                         if noserial:
 123                                 raise Exception("No PHPSerializer module available")
 124                         s = PHPUnserialize()
 125                         o = s.unserialize(f.read())
 126                 f.close()
 127                 return o
 128
 129
 130         # use the environment to extract the data associated with the local
 131         # variable 'name'
 132         def dump(self, name, obj=None, type=None, depth=1):
 133                 if obj == None:
 134                         o = inspect.getouterframes(inspect.currentframe())
 135                         up1 = o[depth][0] # get the frame one prior to (up from) this frame
 136                         argvals = inspect.getargvalues(up1)
 137                         # TODO: check that 'name' is a local variable; otherwise this would fail.
 138                         obj = argvals[3][name] # extract the local variable name 'name'
 139                 if not os.path.isdir("%s/" % self.path):
 140                         os.mkdir("%s" % self.path)
 141                 if config.debug:
 142                         name = "debug.%s" % name
 143                 else:
 144                         name = "production.%s" % name
 145                 f = open(self.__file(name, type), 'w')
 146                 if type == None:
 147                         pickle.dump(obj, f)
 148                 else:
 149                         if noserial:
 150                                 raise Exception("No PHPSerializer module available")
 151                         s = PHPSerialize()
 152                         f.write(s.serialize(obj))
 153                 f.close()
 154                 return
 155
 156
 157 COMMAND_TIMEOUT = 60
 158 ssh_options = { 'StrictHostKeyChecking':'no',
 159                                 'BatchMode':'yes',
 160                                 'PasswordAuthentication':'no',
 161                                 'ConnectTimeout':'%s' % COMMAND_TIMEOUT}
 162 from select import select
 163 import subprocess
 164 import signal
 165
 166 class Sopen(subprocess.Popen):
 167         def kill(self, signal = signal.SIGTERM):
 168                 os.kill(self.pid, signal)
 169
 170 def read_t(stream, count, timeout=COMMAND_TIMEOUT*2):
 171         lin, lout, lerr = select([stream], [], [], timeout)
 172         if len(lin) == 0:
 173                 raise ExceptionTimeout("TIMEOUT Running: %s" % cmd)
 174
 175         return stream.read(count)
 176
 177 class CMD:
 178         def __init__(self):
 179                 pass
 180
 181         def run_noexcept(self, cmd, timeout=COMMAND_TIMEOUT*2):
 182
 183                 #print "CMD.run_noexcept(%s)" % cmd
 184                 try:
 185                         return CMD.run(self,cmd,timeout)
 186                 except ExceptionTimeout:
 187                         import traceback; print traceback.print_exc()
 188                         return ("", "SCRIPTTIMEOUT")
 189
 190         def system(self, cmd, timeout=COMMAND_TIMEOUT*2):
 191                 (o,e) = self.run(cmd, timeout)
 192                 self.output = o
 193                 self.error = e
 194                 if self.s.returncode is None:
 195                         self.s.wait()
 196                 return self.s.returncode
 197
 198         def run(self, cmd, timeout=COMMAND_TIMEOUT*2):
 199
 200                 #print "CMD.run(%s)" % cmd
 201                 s = Sopen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
 202                 self.s = s
 203                 (f_in, f_out, f_err) = (s.stdin, s.stdout, s.stderr)
 204                 #print "calling select(%s)" % timeout
 205                 lout, lin, lerr = select([f_out], [], [f_err], timeout)
 206                 #print "TIMEOUT!!!!!!!!!!!!!!!!!!!"
 207                 if len(lin) == 0 and len(lout) == 0 and len(lerr) == 0:
 208                         # Reached a timeout!  Nuke process so it does not hang.
 209                         #print "KILLING"
 210                         s.kill(signal.SIGKILL)
 211                         raise ExceptionTimeout("TIMEOUT Running: %s" % cmd)
 212                 else:
 213                         #print "RETURNING"
 214                         #print len(lin), len(lout), len(lerr)
 215                         pass
 216
 217                 o_value = ""
 218                 e_value = ""
 219
 220                 #print "reading from f_out"
 221                 if len(lout) > 0: o_value = f_out.read()
 222                 #print "reading from f_err"
 223                 if len(lerr) > 0: e_value = f_err.read()
 224
 225                 #print "striping output"
 226                 o_value = o_value.strip()
 227                 e_value = e_value.strip()
 228
 229                 #print "OUTPUT", o_value, e_value
 230
 231                 #print "closing files"
 232                 f_out.close()
 233                 f_in.close()
 234                 f_err.close()
 235                 try:
 236                         #print "s.kill()"
 237                         s.kill()
 238                         #print "after s.kill()"
 239                 except OSError:
 240                         # no such process, due to it already exiting...
 241                         pass
 242
 243                 #print o_value, e_value
 244                 return (o_value, e_value)
 245
 246         def runargs(self, args, timeout=COMMAND_TIMEOUT*2):
 247
 248                 #print "CMD.run(%s)" % " ".join(args)
 249                 s = Sopen(args, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
 250                 self.s = s
 251                 (f_in, f_out, f_err) = (s.stdin, s.stdout, s.stderr)
 252                 lout, lin, lerr = select([f_out], [], [f_err], timeout)
 253                 if len(lin) == 0 and len(lout) == 0 and len(lerr) == 0:
 254                         # Reached a timeout!  Nuke process so it does not hang.
 255                         s.kill(signal.SIGKILL)
 256                         raise ExceptionTimeout("TIMEOUT Running: %s" % cmd)
 257                 o_value = f_out.read()
 258                 e_value = ""
 259                 if o_value == "":       # An error has occured
 260                         e_value = f_err.read()
 261
 262                 o_value = o_value.strip()
 263                 e_value = e_value.strip()
 264
 265                 f_out.close()
 266                 f_in.close()
 267                 f_err.close()
 268                 try:
 269                         s.kill()
 270                 except OSError:
 271                         # no such process, due to it already exiting...
 272                         pass
 273
 274                 return (o_value, e_value)
 275
 276
 277 class SSH(CMD):
 278         def __init__(self, user, host, port=22, options = ssh_options):
 279                 self.options = options
 280                 self.user = user
 281                 self.host = host
 282                 self.port = port
 283                 return
 284
 285         def __options_to_str(self):
 286                 options = ""
 287                 for o,v in self.options.iteritems():
 288                         options = options + "-o %s=%s " % (o,v)
 289                 return options
 290
 291         def run(self, cmd, timeout=COMMAND_TIMEOUT*2):
 292                 cmd = "ssh -p %s %s %s@%s '%s'" % (self.port, self.__options_to_str(),
 293                                                                         self.user, self.host, cmd)
 294                 #print "SSH.run(%s)" % cmd
 295                 return CMD.run(self, cmd, timeout)
 296
 297         def get_file(self, rmt_filename, local_filename=None):
 298                 if local_filename == None:
 299                         local_filename = "./"
 300                 cmd = "scp -P %s -B %s %s@%s:%s %s" % (self.port, self.__options_to_str(),
 301                                                                         self.user, self.host,
 302                                                                         rmt_filename, local_filename)
 303                 # output :
 304                 #       errors will be on stderr,
 305                 #   success will have a blank stderr...
 306                 return CMD.run_noexcept(self, cmd)
 307
 308         def run_noexcept(self, cmd):
 309                 cmd = "ssh -p %s %s %s@%s '%s'" % (self.port, self.__options_to_str(),
 310                                                                         self.user, self.host, cmd)
 311                 #print "SSH.run_noexcept(%s)" % cmd
 312                 return CMD.run_noexcept(self, cmd)
 313
 314         def run_noexcept2(self, cmd, timeout=COMMAND_TIMEOUT*2):
 315                 cmd = "ssh -p %s %s %s@%s %s" % (self.port, self.__options_to_str(),
 316                                                                         self.user, self.host, cmd)
 317                 #print "SSH.run_noexcept2(%s)" % cmd
 318                 r = CMD.run_noexcept(self, cmd, timeout)
 319
 320                 # XXX: this may be resulting in deadlocks... not sure.
 321                 #if self.s.returncode is None:
 322                 #       #self.s.kill()
 323                 #       self.s.kill(signal.SIGKILL)
 324                 #       self.s.wait()
 325                 #       self.ret = self.s.returncode
 326                 self.ret = -1
 327
 328                 return r
 329
 330         def system2(self, cmd, timeout=COMMAND_TIMEOUT*2):
 331                 cmd = "ssh -p %s %s %s@%s %s" % (self.port, self.__options_to_str(),
 332                                                                         self.user, self.host, cmd)
 333                 #print "SSH.system2(%s)" % cmd
 334                 return CMD.system(self, cmd, timeout)
 335
 336         def runE(self, cmd):
 337                 cmd = "ssh -p %s %s %s@%s '%s'" % (self.port, self.__options_to_str(),
 338                                                                         self.user, self.host, cmd)
 339                 if ( DEBUG == 1 ):
 340                         print cmd,
 341                 (f_in, f_out, f_err) = os.popen3(cmd)
 342
 343                 value = f_out.read()
 344                 if value == "": # An error has occured
 345                         value = f_err.read()
 346                         value = value.strip()
 347
 348                 if ( DEBUG == 1 ):
 349                         print " == %s" % value
 350                 f_out.close()
 351                 f_in.close()
 352                 f_err.close()
 353                 return value.strip()
 354
 355 import time
 356 class MyTimer:
 357         def __init__(self):
 358                 self.start = time.time()
 359
 360         def end(self):
 361                 self.end = time.time()
 362                 t = self.end-self.start
 363                 return t
 364
 365         def diff(self):
 366                 self.end = time.time()
 367                 t = self.end-self.start
 368                 self.start = self.end
 369                 return t