bin/mtail.py

   1 #!/usr/bin/env python
   2
   3 '''
   4 Does tail -f on log files in a given directory.
   5 The display is in chronological order of the logged lines,
   6 given that the first column of log files is timestamp.
   7 It can be altered to fit other formats too
   8 '''
   9
  10 import os, sys, time
  11 import select
  12 import sys, tty, termios
  13
  14 from optparse import OptionParser
  15
  16 class mtail:
  17
  18     subversion_id = "$Id$"
  19
  20     default_time_format = "%H:%M:%S"
  21
  22     def __init__ (self, args ):
  23
  24         # internal structure for tracking changes
  25         self.files = {}
  26         # parse command-line args : will set options and args
  27         self.parse_args(args)
  28         # initialize
  29         self.scan_files()
  30
  31     def parse_args (self, args):
  32         usage = """usage: %prog [options] file-or-dir ...
  33 example:
  34 # %prog -e '*access*' /var/log"""
  35         parser=OptionParser(usage=usage,version=self.subversion_id)
  36         # tail_period
  37         parser.add_option("-p","--period", type="int", dest="tail_period", default=1,
  38                           help="Files check period in seconds")
  39         # rescan_period
  40         parser.add_option("-d","--dir-period", type="int", dest="rescan_period", default=5,
  41                           help="Directories rescan period in seconds")
  42         # time format
  43         parser.add_option("-f","--format", dest="time_format", default=mtail.default_time_format,
  44                           help="Time format, defaults to " + mtail.default_time_format)
  45         # show time
  46         parser.add_option("-r","--raw", action="store_true", dest="show_time", default=True,
  47                           help="Suppresses time display")
  48
  49         # note for exclusion patterns
  50         parser.add_option("-e","--exclude", action="append", dest="excludes", default=[],
  51                           help="Exclusion pattern  -- can be specified multiple times applies on files not explicitly mentioned on the command-line")
  52
  53         parser.add_option("-u","--usual",action="store_true",dest="plc_mode",default=False,
  54                           help="Shortcut for watching /var/log with default settings")
  55         parser.add_option("-s","--sfa",action="store_true",dest="sfa_mode",default=False,
  56                           help="Shortcut for monitoring server-side SFA log files")
  57
  58         # verbosity
  59         parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
  60                           help="Run in verbose mode")
  61
  62         (self.options, self.args) = parser.parse_args(args)
  63         self.option_parser = parser
  64
  65         ### plc shortcuts
  66         if self.options.plc_mode:
  67             # monitor all files in /var/log with some exceptions
  68             self.options.excludes.append('*access_log')
  69             self.options.excludes.append('*request_log')
  70             self.options.excludes.append('*.swp')
  71             self.args.append('/var/log')
  72             # watch the postgresql logs as well
  73             self.args.append('/var/lib/pgsql/data/pg_log')
  74
  75         if self.options.sfa_mode:
  76             self.args.append("/var/log/sfa.log")
  77             self.args.append("/var/log/sfa_import.log")
  78             self.args.append("/var/log/httpd/sfa_access_log")
  79
  80         if self.options.verbose:
  81             print 'Version:',self.subversion_id
  82             print 'Options:',self.options
  83             print 'Arguments:',self.args
  84
  85     def file_size (self,filename):
  86         try:
  87             return os.stat(filename)[6]
  88         except:
  89             print "WARNING: file %s has vanished"%filename
  90             return 0
  91
  92     def number_files (self):
  93         return len(self.files)
  94
  95     # scans given arguments, and updates files accordingly
  96     # can be run several times
  97     def scan_files (self) :
  98
  99         if self.options.verbose:
 100             print 'entering scan_files, files=',self.files
 101
 102         # mark entries in files as pre-existing
 103         for key in self.files:
 104             self.files[key]['old-file']=True
 105
 106         # refreshes the proper set of filenames
 107         filenames = []
 108         for arg in self.args:
 109             if self.options.verbose:
 110                 print 'scan_files -- Considering arg',arg
 111             if os.path.isfile (arg):
 112                 filenames += [ arg ]
 113             elif os.path.isdir (arg) :
 114                 filenames += self.walk (arg)
 115             else:
 116                 print "mtail : no such file or directory %s -- ignored"%arg
 117
 118         # updates files
 119         for filename in filenames :
 120             # known file
 121             if self.files.has_key(filename):
 122                 size = self.file_size(filename)
 123                 offset = self.files[filename]['size']
 124                 if size > offset:
 125                     self.show_file_end(filename,offset,size)
 126                     self.files[filename]['size']=size
 127                 elif size < offset:
 128                     self.show_file_when_size_decreased(filename,offset,size)
 129                 try:
 130                     del self.files[filename]['old-file']
 131                 except:
 132                     pass
 133             else:
 134                 # enter file with current size
 135                 # if we didn't set format yet, it's because we are initializing
 136                 try:
 137                     self.format
 138                     self.show_now()
 139                     print self.format%filename,"new file"
 140                     self.show_file_end(filename,0,self.file_size(filename))
 141                 except:
 142                     pass
 143                 self.files[filename]={'size':self.file_size(filename)}
 144
 145         # cleanup
 146         # avoid side-effects on the current loop basis
 147         read_filenames = self.files.keys()
 148         for filename in read_filenames:
 149             if self.files[filename].has_key('old-file'):
 150                 self.show_now()
 151                 print self.format%filename,"file has gone"
 152                 del self.files[filename]
 153
 154         # compute margin and format
 155         if not filenames:
 156             print sys.argv[0],": WARNING : no file in scope"
 157             self.format="%s"
 158         else:
 159             if len(filenames)==1:
 160                 self.margin=len(filenames[0])
 161             else:
 162                 # this stupidly fails when there's only 1 file
 163                 self.margin=max(*[len(f) for f in filenames])
 164             self.format="%%%ds"%self.margin
 165             if self.options.verbose:
 166                 print 'Current set of files:',filenames
 167
 168     def tail_files (self):
 169
 170         if self.options.verbose:
 171             print 'tail_files'
 172         for filename in self.files:
 173             size = self.file_size(filename)
 174             offset = self.files[filename]['size']
 175             if size != offset:
 176                 self.show_file_end(filename,offset,size)
 177                 self.files[filename]['size']=size
 178
 179     def show_now (self):
 180         if self.options.show_time:
 181             label=time.strftime(self.options.time_format,time.localtime())
 182             print label,
 183
 184     def show_file_end (self, filename, offset, size):
 185         try:
 186             file = open(filename,"r")
 187         # file has vanished
 188         except:
 189             return
 190         file.seek(offset)
 191         line=file.read(size-offset)
 192         self.show_now()
 193         print self.format%filename,'----------------------------------------'
 194         print line
 195         file.close()
 196
 197     def show_file_when_size_decreased (self, filename, offset, size):
 198         print self.format%filename,'---------- file size decreased ---------',
 199         if self.options.verbose:
 200             print 'size during last check',offset,'current size',size
 201         else:
 202             print ''
 203
 204     # get all files under a directory
 205     def walk ( self, root ):
 206         import fnmatch, os, string
 207
 208         # initialize
 209         result = []
 210
 211         # must have at least root folder
 212         try:
 213             names = os.listdir(root)
 214         except os.error:
 215             return result
 216
 217         # check each file
 218         for name in names:
 219             fullname = os.path.normpath(os.path.join(root, name))
 220
 221             # a file : check for excluded, otherwise append
 222             if os.path.isfile(fullname):
 223                 try:
 224                     for exclude in self.options.excludes:
 225                         if fnmatch.fnmatch(name, exclude):
 226                             raise Exception('excluded')
 227                     result.append(fullname)
 228                 except:
 229                     pass
 230             # a dir : let's recurse - avoid symlinks for anti-loop
 231             elif os.path.isdir(fullname) and not os.path.islink(fullname):
 232                 result = result + self.walk( fullname )
 233
 234         return result
 235
 236     def run (self):
 237
 238         if len(self.args) == 0:
 239             self.option_parser.print_help()
 240             sys.exit(1)
 241         counter = 0
 242
 243         fdin = sys.stdin.fileno()
 244         while True:
 245             ## hit the period ?
 246             # dont do this twice at startup
 247             if (counter !=0 and counter % self.options.rescan_period == 0):
 248                 self.scan_files()
 249
 250             if (counter % self.options.tail_period == 0):
 251                 self.tail_files()
 252
 253             time.sleep(1)
 254             counter += 1
 255             # react on some keys - rough but convenient
 256             if os.isatty(fdin):
 257                 # read stdin
 258                 typed=[]
 259                 while select.select([sys.stdin,],[],[],0.0)[0]:
 260                     typed.append(sys.stdin.read(1))
 261 #                print 'found chars',typed
 262                 for char in typed:
 263                     if char.lower() in ['l']: os.system("clear")
 264                     elif char.lower() in ['m']:
 265                         for i in range(3) : print 60 * '='
 266                     elif char.lower() in ['q']: sys.exit(0)
 267                     elif char.lower() in ['h']:
 268                         print """l: refresh page
 269 m: mark
 270 q: quit
 271 h: help"""
 272
 273 ###
 274 if __name__ == '__main__':
 275     mtail (sys.argv[1:]).run()