add email_exception() to all except: statements.

[monitor.git] / findbad.py

diff --git a/findbad.py b/findbad.py
index 47459ad..630f1c5 100755 (executable)
--- a/findbad.py
+++ b/findbad.py
@@ -56,6 +56,8 @@ def collectPingAndSSH(nodename, cohash):
                                echo '  "bmlog":"'`ls /tmp/bm.log`'",'
                                echo '  "bootcd":"'`cat /mnt/cdrom/bootme/ID`'",'
                                echo '  "nm":"'`ps ax | grep nm.py | grep -v grep`'",'
+                               echo '  "readonlyfs":"'`touch /var/log/monitor 2>&1`'",'
+                               echo '  "dns":"'`host boot.planet-lab.org 2>&1`'",'
                                echo '  "princeton_comon":"'`ls -d /vservers/princeton_comon`'",'
 
                                ID=`grep princeton_comon /etc/passwd | awk -F : '{if ( $3 > 500 ) { print $3}}'` 
@@ -70,11 +72,17 @@ EOF                 """)
                                values['sshport'] = port
                                break
                        else:
-                               values.update({'kernel': "", 'bmlog' : "", 'bootcd' : '', 'nm' :
-                               '', 'princeton_comon' : '', 'princeton_comon_running' : '',
-                               'princeton_comon_procs' : '', 'sshport' : None})
+                               values.update({'kernel': "", 'bmlog' : "", 'bootcd' : '', 
+                                                               'nm' : '', 
+                                                               'readonlyfs' : '',
+                                                               'dns' : '',
+                                                               'princeton_comon' : '', 
+                                                               'princeton_comon_running' : '', 
+                                                               'princeton_comon_procs' : '', 'sshport' : None})
        except:
                print traceback.print_exc()
+               from nodecommon import email_exception
+               email_exception()
                sys.exit(1)
 
        ### RUN SSH ######################
@@ -87,14 +95,14 @@ EOF                 """)
        oval = values['kernel']
        if "2.6.17" in oval or "2.6.2" in oval:
                values['ssh'] = 'SSH'
-               values['category'] = 'ALPHA'
+               values['category'] = 'PROD'
                if "bm.log" in values['bmlog']:
                        values['state'] = 'DEBUG'
                else:
                        values['state'] = 'BOOT'
        elif "2.6.12" in oval or "2.6.10" in oval:
                values['ssh'] = 'SSH'
-               values['category'] = 'PROD'
+               values['category'] = 'OLDPROD'
                if "bm.log" in values['bmlog']:
                        values['state'] = 'DEBUG'
                else:
@@ -197,6 +205,8 @@ EOF                 """)
        except:
                b_except = True
                traceback.print_exc()
+               from nodecommon import email_exception
+               email_exception()
 
        plc_lock.release()
        if b_except: return (None, None)
@@ -234,6 +244,8 @@ EOF                 """)
        except:
                b_except = True
                traceback.print_exc()
+               from nodecommon import email_exception
+               email_exception()
 
        plc_lock.release()
        if b_except: return (None, None)
@@ -391,6 +403,8 @@ if __name__ == '__main__':
                main()
        except Exception, err:
                print traceback.print_exc()
+               from nodecommon import email_exception
+               email_exception()
                print "Exception: %s" % err
                print "Saving data... exitting."
                database.dbDump(config.dbname, externalState)