moncommands.py

   1 import os
   2 import fcntl
   3 import traceback
   4
   5 DEBUG= 0
   6
   7 class ExceptionTimeout(Exception): pass
   8 class ExceptionReadTimeout(Exception): pass
   9 COMMAND_TIMEOUT = 60
  10 ssh_options = { 'StrictHostKeyChecking':'no',
  11                                 'BatchMode':'yes',
  12                                 'PasswordAuthentication':'no',
  13                                 'ConnectTimeout':'%s' % COMMAND_TIMEOUT}
  14 from select import select
  15 import subprocess
  16 import signal
  17
  18 class Sopen(subprocess.Popen):
  19         def kill(self, sig = signal.SIGTERM):
  20                 try:
  21                         # NOTE: this also kills parent... so doesn't work like I want.
  22                         # NOTE: adding 'exec' before the cmd removes the extra sh, and
  23                         #               partially addresses this problem.
  24                         #os.killpg(os.getpgid(self.pid), signal.SIGKILL)
  25                         os.kill(self.pid, sig)
  26                 except OSError:
  27                         # no such process, due to it already exiting...
  28                         pass
  29
  30
  31 def read_t(stream, count=1, timeout=COMMAND_TIMEOUT*2):
  32         if count == 1:
  33                 retstr = ""
  34
  35                 while True:
  36                         lin, lout, lerr = select([stream], [], [], timeout)
  37                         if len(lin) == 0:
  38                                 print "timeout!"
  39                                 raise ExceptionReadTimeout("TIMEOUT reading from command")
  40
  41                         try:
  42                                 outbytes = stream.read(count)
  43                         except IOError, err:
  44                                 print 'no content yet.'
  45                                 # due to no content.
  46                                 # the select timeout should catch this.
  47                                 continue
  48
  49                         if not outbytes:
  50                                 break
  51                         retstr += outbytes
  52
  53                 return retstr
  54         else:
  55                 lin, lout, lerr = select([stream], [], [], timeout)
  56                 if len(lin) == 0:
  57                         raise ExceptionReadTimeout("TIMEOUT reading from command")
  58
  59                 return stream.read(count)
  60
  61 class CMD:
  62         def __init__(self):
  63                 pass
  64
  65         def run_noexcept(self, cmd, timeout=COMMAND_TIMEOUT*2):
  66
  67                 try:
  68                         return CMD.run(self,cmd,timeout)
  69                 except ExceptionTimeout:
  70                         import traceback; print traceback.print_exc()
  71                         return ("", "ScriptTimeout")
  72                 except ExceptionReadTimeout:
  73                         import traceback
  74                         print traceback.print_exc()
  75                         return ("", "RunningScriptTimeout")
  76                 except Exception, err:
  77                         from nodecommon import email_exception
  78                         email_exception()
  79                         return ("", str(err))
  80
  81         def system(self, cmd, timeout=COMMAND_TIMEOUT*2):
  82                 (o,e) = self.run(cmd, timeout)
  83                 self.output = o
  84                 self.error = e
  85                 if self.s.returncode is None:
  86                         self.s.wait()
  87                 return self.s.returncode
  88
  89         def run(self, cmd, timeout=COMMAND_TIMEOUT*2):
  90
  91                 print "CMD.run(%s)" % cmd
  92                 s = Sopen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
  93                 self.s = s
  94                 (f_in, f_out, f_err) = (s.stdin, s.stdout, s.stderr)
  95                 lout, lin, lerr = select([f_out], [], [f_err], timeout)
  96                 if len(lin) == 0 and len(lout) == 0 and len(lerr) == 0:
  97                         # Reached a timeout!  Nuke process so it does not hang.
  98                         print "TIMEOUT!!!!!!!!!!!!!!!!!!!"
  99                         s.kill(signal.SIGKILL)
 100                         raise ExceptionTimeout("TIMEOUT Running: %s" % cmd)
 101                 else:
 102                         #print "RETURNING"
 103                         #print len(lin), len(lout), len(lerr)
 104                         pass
 105
 106                 o_value = ""
 107                 e_value = ""
 108
 109                 #print "reading from f_out"
 110                 #if len(lout) > 0: o_value = f_out.read()
 111                 #print "reading from f_err"
 112                 #if len(lerr) > 0: e_value = f_err.read()
 113                 #o_value = f_out.read()
 114                 flags = fcntl.fcntl(f_out, fcntl.F_GETFL)
 115                 fcntl.fcntl(f_out, fcntl.F_SETFL, flags | os.O_NONBLOCK)
 116
 117                 try:
 118                         o_value = read_t(f_out,1,30)
 119                 except ExceptionReadTimeout:
 120                         s.kill(signal.SIGKILL)
 121                         raise ExceptionReadTimeout("TIMEOUT: failed to read from cmd: %s" % cmd)
 122
 123                 e_value = f_err.read()
 124
 125                 o_value = o_value.strip()
 126                 e_value = e_value.strip()
 127
 128                 f_out.close()
 129                 f_in.close()
 130                 f_err.close()
 131                 s.kill(signal.SIGKILL)
 132
 133                 return (o_value, e_value)
 134
 135         def runargs(self, args, timeout=COMMAND_TIMEOUT*2):
 136
 137                 #print "CMD.run(%s)" % " ".join(args)
 138                 s = Sopen(args, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
 139                 self.s = s
 140                 (f_in, f_out, f_err) = (s.stdin, s.stdout, s.stderr)
 141                 lout, lin, lerr = select([f_out], [], [f_err], timeout)
 142                 if len(lin) == 0 and len(lout) == 0 and len(lerr) == 0:
 143                         # Reached a timeout!  Nuke process so it does not hang.
 144                         s.kill(signal.SIGKILL)
 145                         raise ExceptionTimeout("TIMEOUT Running: %s" % cmd)
 146                 o_value = f_out.read()
 147                 e_value = ""
 148                 if o_value == "":       # An error has occured
 149                         e_value = f_err.read()
 150
 151                 o_value = o_value.strip()
 152                 e_value = e_value.strip()
 153
 154                 f_out.close()
 155                 f_in.close()
 156                 f_err.close()
 157                 s.kill(signal.SIGKILL)
 158
 159                 return (o_value, e_value)
 160
 161
 162 class SSH(CMD):
 163         def __init__(self, user, host, port=22, options = ssh_options):
 164                 self.options = options
 165                 self.user = user
 166                 self.host = host
 167                 self.port = port
 168                 return
 169
 170         def __options_to_str(self):
 171                 options = ""
 172                 for o,v in self.options.iteritems():
 173                         options = options + "-o %s=%s " % (o,v)
 174                 return options
 175
 176         def run(self, cmd, timeout=COMMAND_TIMEOUT*2):
 177                 cmd = "ssh -p %s %s %s@%s '%s'" % (self.port, self.__options_to_str(),
 178                                                                         self.user, self.host, cmd)
 179                 #print "SSH.run(%s)" % cmd
 180                 return CMD.run(self, cmd, timeout)
 181
 182         def get_file(self, rmt_filename, local_filename=None):
 183                 if local_filename == None:
 184                         local_filename = "./"
 185                 cmd = "scp -P %s -B %s %s@%s:%s %s" % (self.port, self.__options_to_str(),
 186                                                                         self.user, self.host,
 187                                                                         rmt_filename, local_filename)
 188                 # output :
 189                 #       errors will be on stderr,
 190                 #   success will have a blank stderr...
 191                 return CMD.run_noexcept(self, cmd)
 192
 193         def run_noexcept(self, cmd):
 194                 cmd = "ssh -p %s %s %s@%s '%s'" % (self.port, self.__options_to_str(),
 195                                                                         self.user, self.host, cmd)
 196                 #print "SSH.run_noexcept(%s)" % cmd
 197                 return CMD.run_noexcept(self, cmd)
 198
 199         def run_noexcept2(self, cmd, timeout=COMMAND_TIMEOUT*2):
 200                 cmd = "exec ssh -p %s %s %s@%s %s" % (self.port, self.__options_to_str(),
 201                                                                         self.user, self.host, cmd)
 202                 r = CMD.run_noexcept(self, cmd, timeout)
 203                 self.ret = -1
 204
 205                 return r
 206
 207         def system2(self, cmd, timeout=COMMAND_TIMEOUT*2):
 208                 cmd = "ssh -p %s %s %s@%s %s" % (self.port, self.__options_to_str(),
 209                                                                         self.user, self.host, cmd)
 210                 #print "SSH.system2(%s)" % cmd
 211                 return CMD.system(self, cmd, timeout)
 212
 213         def runE(self, cmd):
 214                 cmd = "ssh -p %s %s %s@%s '%s'" % (self.port, self.__options_to_str(),
 215                                                                         self.user, self.host, cmd)
 216                 if ( DEBUG == 1 ):
 217                         print cmd,
 218                 (f_in, f_out, f_err) = os.popen3(cmd)
 219
 220                 value = f_out.read()
 221                 if value == "": # An error has occured
 222                         value = f_err.read()
 223                         value = value.strip()
 224
 225                 if ( DEBUG == 1 ):
 226                         print " == %s" % value
 227                 f_out.close()
 228                 f_in.close()
 229                 f_err.close()
 230                 return value.strip()
 231
 232 import time
 233 class MyTimer:
 234         def __init__(self):
 235                 self.start = time.time()
 236
 237         def end(self):
 238                 self.end = time.time()
 239                 t = self.end-self.start
 240                 return t
 241
 242         def diff(self):
 243                 self.end = time.time()
 244                 t = self.end-self.start
 245                 self.start = self.end
 246                 return t