moncommands.py

   1 import os
   2 import fcntl
   3
   4 DEBUG= 0
   5
   6 class ExceptionTimeout(Exception): pass
   7 class ExceptionReadTimeout(Exception): pass
   8 COMMAND_TIMEOUT = 60
   9 ssh_options = { 'StrictHostKeyChecking':'no',
  10                                 'BatchMode':'yes',
  11                                 'PasswordAuthentication':'no',
  12                                 'ConnectTimeout':'%s' % COMMAND_TIMEOUT}
  13 from select import select
  14 import subprocess
  15 import signal
  16
  17 class Sopen(subprocess.Popen):
  18         def kill(self, sig = signal.SIGTERM):
  19                 try:
  20                         # NOTE: this also kills parent... so doesn't work like I want.
  21                         # NOTE: adding 'exec' before the cmd removes the extra sh, and
  22                         #               partially addresses this problem.
  23                         #os.killpg(os.getpgid(self.pid), signal.SIGKILL)
  24                         os.kill(self.pid, sig)
  25                 except OSError:
  26                         # no such process, due to it already exiting...
  27                         pass
  28
  29
  30 def read_t(stream, count=1, timeout=COMMAND_TIMEOUT*2):
  31         if count == 1:
  32                 retstr = ""
  33
  34                 while True:
  35                         lin, lout, lerr = select([stream], [], [], timeout)
  36                         if len(lin) == 0:
  37                                 print "timeout!"
  38                                 raise ExceptionReadTimeout("TIMEOUT reading from command")
  39
  40                         try:
  41                                 outbytes = stream.read(count)
  42                         except IOError, err:
  43                                 print 'no content yet.'
  44                                 # due to no content.
  45                                 # the select timeout should catch this.
  46                                 continue
  47
  48                         if not outbytes:
  49                                 break
  50                         retstr += outbytes
  51
  52                 return retstr
  53         else:
  54                 lin, lout, lerr = select([stream], [], [], timeout)
  55                 if len(lin) == 0:
  56                         raise ExceptionReadTimeout("TIMEOUT reading from command")
  57
  58                 return stream.read(count)
  59
  60 class CMD:
  61         def __init__(self):
  62                 pass
  63
  64         def run_noexcept(self, cmd, timeout=COMMAND_TIMEOUT*2):
  65
  66                 try:
  67                         return CMD.run(self,cmd,timeout)
  68                 except ExceptionTimeout:
  69                         import traceback; print traceback.print_exc()
  70                         return ("", "ScriptTimeout")
  71                 except ExceptionReadTimeout:
  72                         print traceback.print_exc()
  73                         return ("", "RunningScriptTimeout")
  74                 except Exception, err:
  75                         from nodecommon import email_exception
  76                         email_exception()
  77                         return ("", str(err))
  78
  79         def system(self, cmd, timeout=COMMAND_TIMEOUT*2):
  80                 (o,e) = self.run(cmd, timeout)
  81                 self.output = o
  82                 self.error = e
  83                 if self.s.returncode is None:
  84                         self.s.wait()
  85                 return self.s.returncode
  86
  87         def run(self, cmd, timeout=COMMAND_TIMEOUT*2):
  88
  89                 print "CMD.run(%s)" % cmd
  90                 s = Sopen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
  91                 self.s = s
  92                 (f_in, f_out, f_err) = (s.stdin, s.stdout, s.stderr)
  93                 lout, lin, lerr = select([f_out], [], [f_err], timeout)
  94                 if len(lin) == 0 and len(lout) == 0 and len(lerr) == 0:
  95                         # Reached a timeout!  Nuke process so it does not hang.
  96                         print "TIMEOUT!!!!!!!!!!!!!!!!!!!"
  97                         s.kill(signal.SIGKILL)
  98                         raise ExceptionTimeout("TIMEOUT Running: %s" % cmd)
  99                 else:
 100                         #print "RETURNING"
 101                         #print len(lin), len(lout), len(lerr)
 102                         pass
 103
 104                 o_value = ""
 105                 e_value = ""
 106
 107                 #print "reading from f_out"
 108                 #if len(lout) > 0: o_value = f_out.read()
 109                 #print "reading from f_err"
 110                 #if len(lerr) > 0: e_value = f_err.read()
 111                 #o_value = f_out.read()
 112                 flags = fcntl.fcntl(f_out, fcntl.F_GETFL)
 113                 fcntl.fcntl(f_out, fcntl.F_SETFL, flags | os.O_NONBLOCK)
 114
 115                 try:
 116                         o_value = read_t(f_out,1,30)
 117                 except ExceptionReadTimeout:
 118                         s.kill(signal.SIGKILL)
 119                         raise ExceptionReadTimeout("TIMEOUT: failed to read from cmd: %s" % cmd)
 120
 121                 e_value = f_err.read()
 122
 123                 o_value = o_value.strip()
 124                 e_value = e_value.strip()
 125
 126                 f_out.close()
 127                 f_in.close()
 128                 f_err.close()
 129                 s.kill(signal.SIGKILL)
 130
 131                 return (o_value, e_value)
 132
 133         def runargs(self, args, timeout=COMMAND_TIMEOUT*2):
 134
 135                 #print "CMD.run(%s)" % " ".join(args)
 136                 s = Sopen(args, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
 137                 self.s = s
 138                 (f_in, f_out, f_err) = (s.stdin, s.stdout, s.stderr)
 139                 lout, lin, lerr = select([f_out], [], [f_err], timeout)
 140                 if len(lin) == 0 and len(lout) == 0 and len(lerr) == 0:
 141                         # Reached a timeout!  Nuke process so it does not hang.
 142                         s.kill(signal.SIGKILL)
 143                         raise ExceptionTimeout("TIMEOUT Running: %s" % cmd)
 144                 o_value = f_out.read()
 145                 e_value = ""
 146                 if o_value == "":       # An error has occured
 147                         e_value = f_err.read()
 148
 149                 o_value = o_value.strip()
 150                 e_value = e_value.strip()
 151
 152                 f_out.close()
 153                 f_in.close()
 154                 f_err.close()
 155                 s.kill(signal.SIGKILL)
 156
 157                 return (o_value, e_value)
 158
 159
 160 class SSH(CMD):
 161         def __init__(self, user, host, port=22, options = ssh_options):
 162                 self.options = options
 163                 self.user = user
 164                 self.host = host
 165                 self.port = port
 166                 return
 167
 168         def __options_to_str(self):
 169                 options = ""
 170                 for o,v in self.options.iteritems():
 171                         options = options + "-o %s=%s " % (o,v)
 172                 return options
 173
 174         def run(self, cmd, timeout=COMMAND_TIMEOUT*2):
 175                 cmd = "ssh -p %s %s %s@%s '%s'" % (self.port, self.__options_to_str(),
 176                                                                         self.user, self.host, cmd)
 177                 #print "SSH.run(%s)" % cmd
 178                 return CMD.run(self, cmd, timeout)
 179
 180         def get_file(self, rmt_filename, local_filename=None):
 181                 if local_filename == None:
 182                         local_filename = "./"
 183                 cmd = "scp -P %s -B %s %s@%s:%s %s" % (self.port, self.__options_to_str(),
 184                                                                         self.user, self.host,
 185                                                                         rmt_filename, local_filename)
 186                 # output :
 187                 #       errors will be on stderr,
 188                 #   success will have a blank stderr...
 189                 return CMD.run_noexcept(self, cmd)
 190
 191         def run_noexcept(self, cmd):
 192                 cmd = "ssh -p %s %s %s@%s '%s'" % (self.port, self.__options_to_str(),
 193                                                                         self.user, self.host, cmd)
 194                 #print "SSH.run_noexcept(%s)" % cmd
 195                 return CMD.run_noexcept(self, cmd)
 196
 197         def run_noexcept2(self, cmd, timeout=COMMAND_TIMEOUT*2):
 198                 cmd = "exec ssh -p %s %s %s@%s %s" % (self.port, self.__options_to_str(),
 199                                                                         self.user, self.host, cmd)
 200                 r = CMD.run_noexcept(self, cmd, timeout)
 201                 self.ret = -1
 202
 203                 return r
 204
 205         def system2(self, cmd, timeout=COMMAND_TIMEOUT*2):
 206                 cmd = "ssh -p %s %s %s@%s %s" % (self.port, self.__options_to_str(),
 207                                                                         self.user, self.host, cmd)
 208                 #print "SSH.system2(%s)" % cmd
 209                 return CMD.system(self, cmd, timeout)
 210
 211         def runE(self, cmd):
 212                 cmd = "ssh -p %s %s %s@%s '%s'" % (self.port, self.__options_to_str(),
 213                                                                         self.user, self.host, cmd)
 214                 if ( DEBUG == 1 ):
 215                         print cmd,
 216                 (f_in, f_out, f_err) = os.popen3(cmd)
 217
 218                 value = f_out.read()
 219                 if value == "": # An error has occured
 220                         value = f_err.read()
 221                         value = value.strip()
 222
 223                 if ( DEBUG == 1 ):
 224                         print " == %s" % value
 225                 f_out.close()
 226                 f_in.close()
 227                 f_err.close()
 228                 return value.strip()
 229
 230 import time
 231 class MyTimer:
 232         def __init__(self):
 233                 self.start = time.time()
 234
 235         def end(self):
 236                 self.end = time.time()
 237                 t = self.end-self.start
 238                 return t
 239
 240         def diff(self):
 241                 self.end = time.time()
 242                 t = self.end-self.start
 243                 self.start = self.end
 244                 return t