keep on notify errors twice a day

diff --git a/scripts/git-mirror.sh b/scripts/git-mirror.sh
index 7768b48..4cd6d34 100755 (executable)
--- a/scripts/git-mirror.sh
+++ b/scripts/git-mirror.sh
@@ -34,8 +34,11 @@ function error () {
     local SUBJECT=$1; shift
     local BODY=$1; shift
 
-    # already notified ? 
-    [ -f $REPO/$NOTIFIED_FILE ] && return
+    # already notified within a half day ?
+    GRACE=$((60*12))
+    is_old=$(find $REPO/$NOTIFIED_FILE -mmin +$GRACE 2> /dev/null)
+    # not there or less than 12 our old: don't notify
+    [ -z "$is_old" ] && return
 
     for admin in $ADMINS; do
         echo -e "$BODY" | mail -s "$SUBJECT" $admin
@@ -57,7 +60,7 @@ function run () {
     ok=true
     if [ -n "$QUIET" ] ; then
        OUTPUT=$($COMMAND 2>&1) || ok=
-       TORECORD="[$REPO] $OUTPUT"
+       TORECORD="[$REPO] $COMMAND\n$OUTPUT"
     else
        echo "[$REPO] Running $COMMAND"
        $COMMAND || ok=
@@ -117,7 +120,6 @@ function mirror_repo () {
     REMOTE_REPO=${REMOTE_GIT}/${GIT_NAME}
     MASTER_REPO=${MASTER_GIT}/${GIT_NAME}
 
-
     # if there is no remote repository it may be that we only have
     # the repository locally and don't need to mirror
     git ls-remote $REMOTE_REPO &> /dev/null || return
@@ -128,7 +130,10 @@ function mirror_repo () {
        run ${REPO_DIR} git fetch origin
        merge_all_branches $REPO_DIR $NAME origin
        if [ $? -ne 0 ]; then
-            error $REPO_DIR "git-mirror.sh failed with module ${NAME}" "Can not fetch from ${MASTER_REPO}" 
+           # format mail body
+           body="Can not fetch from ${MASTER_REPO}\n\n------------\n FAILED COMMANDS:\n"
+           for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
+            error $REPO_DIR "git-mirror.sh failed to merge remote with module ${NAME}" "$body"
        fi
     else
         msg "mirroring ${NAME} for the first time"
@@ -145,16 +150,22 @@ function mirror_repo () {
         STATUS_OUT=$(git status)
         popd > /dev/null
        # format mail body
-       body="STATUS (in $REPO_DIR):\n$STATUS_OUT \n\n------------\n FAILED COMMANDS:\n"
+       body="STATUS in ${REPO_DIR}:\n${STATUS_OUT} \n\n------------\n FAILED COMMANDS:\n"
        for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
-        error $REPO_DIR "git-mirror.sh failed on with module ${NAME}" $body
-    else
-        run ${REPO_DIR} git push --tags local_master
-        push_all_branches $REPO_DIR $NAME local_master origin 
-       
-        # success, remove previous check file if any
-        rm -f $REPO_DIR/$NOTIFIED_FILE
+        error $REPO_DIR "git-mirror.sh failed on with module ${NAME}" "$body"
+       return
+    fi
+    run ${REPO_DIR} git push --tags local_master
+    push_all_branches $REPO_DIR $NAME local_master origin 
+    if [ $? -ne 0 ]; then
+       # format mail body
+       body="FAILED COMMANDS:\n"
+       for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
+        error $REPO_DIR "git-mirror.sh failed to push back with module ${NAME}" "$body"
+       return
     fi
+    # success, remove previous check file if any
+    rm -f $REPO_DIR/$NOTIFIED_FILE
 }
 
 function usage () {
@@ -186,7 +197,7 @@ if [ -f $RUNNING_FILE ] ; then
     exit 1
 fi
 
-trap failure ERR INT
+trap failure INT
 
 shift $((OPTIND-1))
 date > $RUNNING_FILE