mtail.py

   1 #!/usr/bin/env python
   2
   3 '''
   4 Does tail -f on log files in a given directory.
   5 The display is in chronological order of the logged lines,
   6 given that the first column of log files is timestamp.
   7 It can be altered to fit other formats too
   8 '''
   9
  10 import os, sys, time
  11 from optparse import OptionParser
  12
  13 class mtail:
  14
  15     subversion_id = "$Id$"
  16
  17     default_time_format = "%H:%M:%S"
  18
  19     def __init__ (self, args ):
  20
  21         # internal structure for tracking changes
  22         self.files = {}
  23         # parse command-line args : will set options and args
  24         self.parse_args(args)
  25         # initialize
  26         self.scan_files()
  27
  28     def parse_args (self, args):
  29         usage = """usage: %prog [options] file-or-dir ...
  30 example:
  31 # %prog -e '*access*' /var/log"""
  32         parser=OptionParser(usage=usage,version=self.subversion_id)
  33         # tail_period
  34         parser.add_option("-p","--period", type="int", dest="tail_period", default=1,
  35                           help="Files check period in seconds")
  36         # rescan_period
  37         parser.add_option("-d","--dir-period", type="int", dest="rescan_period", default=5,
  38                           help="Directories rescan period in seconds")
  39         # time format
  40         parser.add_option("-f","--format", dest="time_format", default=mtail.default_time_format,
  41                           help="Time format, defaults to " + mtail.default_time_format)
  42         # show time
  43         parser.add_option("-r","--raw", action="store_true", dest="show_time", default=True,
  44                           help="Suppresses time display")
  45
  46         # note for exclusion patterns
  47         parser.add_option("-e","--exclude", action="append", dest="excludes", default=[],
  48                           help="Exclusion pattern  -- can be specified multiple times applies on files not explicitly mentioned on the command-line")
  49
  50         parser.add_option("-u","--usual",action="store_true",dest="plc_mode",default=False,
  51                           help="Shortcut for watching /var/log with default settings")
  52
  53         # verbosity
  54         parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
  55                           help="Run in verbose mode")
  56
  57         (self.options, self.args) = parser.parse_args(args)
  58         self.option_parser = parser
  59
  60         ### plc shortcuts
  61         if self.options.plc_mode:
  62             # monitor all files in /var/log with some exceptions
  63             self.options.excludes.append('*access_log')
  64             self.options.excludes.append('*request_log')
  65             self.options.excludes.append('*.swp')
  66             self.args.append('/var/log')
  67             # watch the postgresql logs as well
  68             self.args.append('/var/lib/pgsql/data/pg_log')
  69
  70         if self.options.verbose:
  71             print 'Version:',self.subversion_id
  72             print 'Options:',self.options
  73             print 'Arguments:',self.args
  74
  75     def file_size (self,filename):
  76         try:
  77             return os.stat(filename)[6]
  78         except:
  79             print "WARNING: file %s has vanished"%filename
  80             return 0
  81
  82     def number_files (self):
  83         return len(self.files)
  84
  85     # scans given arguments, and updates files accordingly
  86     # can be run several times
  87     def scan_files (self) :
  88
  89         if self.options.verbose:
  90             print 'entering scan_files, files=',self.files
  91
  92         # mark entries in files as pre-existing
  93         for key in self.files:
  94             self.files[key]['old-file']=True
  95
  96         # refreshes the proper set of filenames
  97         filenames = []
  98         for arg in self.args:
  99             if self.options.verbose:
 100                 print 'scan_files -- Considering arg',arg
 101             if os.path.isfile (arg):
 102                 filenames += [ arg ]
 103             elif os.path.isdir (arg) :
 104                 filenames += self.walk (arg)
 105             else:
 106                 print "mtail : no such file or directory %s -- ignored"%arg
 107
 108         # updates files
 109         for filename in filenames :
 110             # known file
 111             if self.files.has_key(filename):
 112                 size = self.file_size(filename)
 113                 offset = self.files[filename]['size']
 114                 if size > offset:
 115                     self.show_file_end(filename,offset,size)
 116                     self.files[filename]['size']=size
 117                 elif size < offset:
 118                     self.show_file_when_size_decreased(filename,offset,size)
 119                 try:
 120                     del self.files[filename]['old-file']
 121                 except:
 122                     pass
 123             else:
 124                 # enter file with current size
 125                 # if we didn't set format yet, it's because we are initializing
 126                 try:
 127                     self.format
 128                     self.show_now()
 129                     print self.format%filename,"new file"
 130                     self.show_file_end(filename,0,self.file_size(filename))
 131                 except:
 132                     pass
 133                 self.files[filename]={'size':self.file_size(filename)}
 134
 135         # cleanup
 136         # avoid side-effects on the current loop basis
 137         read_filenames = self.files.keys()
 138         for filename in read_filenames:
 139             if self.files[filename].has_key('old-file'):
 140                 self.show_now()
 141                 print self.format%filename,"file has gone"
 142                 del self.files[filename]
 143
 144         # compute margin and format
 145         if not filenames:
 146             print sys.argv[0],": WARNING : no file in scope"
 147             self.format="%s"
 148         else:
 149             if len(filenames)==1:
 150                 self.margin=len(filenames[0])
 151             else:
 152                 # this stupidly fails when there's only 1 file
 153                 self.margin=max(*[len(f) for f in filenames])
 154             self.format="%%%ds"%self.margin
 155             if self.options.verbose:
 156                 print 'Current set of files:',filenames
 157
 158     def tail_files (self):
 159
 160         if self.options.verbose:
 161             print 'tail_files'
 162         for filename in self.files:
 163             size = self.file_size(filename)
 164             offset = self.files[filename]['size']
 165             if size != offset:
 166                 self.show_file_end(filename,offset,size)
 167                 self.files[filename]['size']=size
 168
 169     def show_now (self):
 170         if self.options.show_time:
 171             label=time.strftime(self.options.time_format,time.localtime())
 172             print label,
 173
 174     def show_file_end (self, filename, offset, size):
 175         try:
 176             file = open(filename,"r")
 177         # file has vanished
 178         except:
 179             return
 180         file.seek(offset)
 181         line=file.read(size-offset)
 182         self.show_now()
 183         print self.format%filename,'----------------------------------------'
 184         print line
 185         file.close()
 186
 187     def show_file_when_size_decreased (self, filename, offset, size):
 188         print self.format%filename,'---------- file size decreased ---------',
 189         if self.options.verbose:
 190             print 'size during last check',offset,'current size',size
 191         else:
 192             print ''
 193
 194     # get all files under a directory
 195     def walk ( self, root ):
 196         import fnmatch, os, string
 197
 198         # initialize
 199         result = []
 200
 201         # must have at least root folder
 202         try:
 203             names = os.listdir(root)
 204         except os.error:
 205             return result
 206
 207         # check each file
 208         for name in names:
 209             fullname = os.path.normpath(os.path.join(root, name))
 210
 211             # a file : check for excluded, otherwise append
 212             if os.path.isfile(fullname):
 213                 try:
 214                     for exclude in self.options.excludes:
 215                         if fnmatch.fnmatch(name, exclude):
 216                             raise Exception('excluded')
 217                     result.append(fullname)
 218                 except:
 219                     pass
 220             # a dir : let's recurse - avoid symlinks for anti-loop
 221             elif os.path.isdir(fullname) and not os.path.islink(fullname):
 222                 result = result + self.walk( fullname )
 223
 224         return result
 225
 226     def run (self):
 227
 228         if len(self.args) == 0:
 229             self.option_parser.print_help()
 230             sys.exit(1)
 231         counter = 0
 232
 233         while 1:
 234             ## hit the period ?
 235             # dont do this twice at startup
 236             if (counter !=0 and counter % self.options.rescan_period == 0):
 237                 self.scan_files()
 238
 239             if (counter % self.options.tail_period == 0):
 240                 self.tail_files()
 241
 242             time.sleep(1)
 243             counter += 1
 244
 245 ###
 246 if __name__ == '__main__':
 247     mtail (sys.argv[1:]).run()