From 24e70167326532a061b23747656a6e7e35869ade Mon Sep 17 00:00:00 2001
From: Thierry Parmentelat <thierry.parmentelat@sophia.inria.fr>
Date: Thu, 13 Oct 2011 13:18:50 +0200
Subject: [PATCH] keep on notify errors twice a day

---
 scripts/git-mirror.sh | 39 +++++++++++++++++++++++++--------------
 1 file changed, 25 insertions(+), 14 deletions(-)

diff --git a/scripts/git-mirror.sh b/scripts/git-mirror.sh
index 7768b48..4cd6d34 100755
--- a/scripts/git-mirror.sh
+++ b/scripts/git-mirror.sh
@@ -34,8 +34,11 @@ function error () {
     local SUBJECT=$1; shift
     local BODY=$1; shift
 
-    # already notified ? 
-    [ -f $REPO/$NOTIFIED_FILE ] && return
+    # already notified within a half day ?
+    GRACE=$((60*12))
+    is_old=$(find $REPO/$NOTIFIED_FILE -mmin +$GRACE 2> /dev/null)
+    # not there or less than 12 our old: don't notify
+    [ -z "$is_old" ] && return
 
     for admin in $ADMINS; do
         echo -e "$BODY" | mail -s "$SUBJECT" $admin
@@ -57,7 +60,7 @@ function run () {
     ok=true
     if [ -n "$QUIET" ] ; then
 	OUTPUT=$($COMMAND 2>&1) || ok=
-	TORECORD="[$REPO] $OUTPUT"
+	TORECORD="[$REPO] $COMMAND\n$OUTPUT"
     else
 	echo "[$REPO] Running $COMMAND"
 	$COMMAND || ok=
@@ -117,7 +120,6 @@ function mirror_repo () {
     REMOTE_REPO=${REMOTE_GIT}/${GIT_NAME}
     MASTER_REPO=${MASTER_GIT}/${GIT_NAME}
 
-
     # if there is no remote repository it may be that we only have
     # the repository locally and don't need to mirror
     git ls-remote $REMOTE_REPO &> /dev/null || return
@@ -128,7 +130,10 @@ function mirror_repo () {
 	run ${REPO_DIR} git fetch origin
 	merge_all_branches $REPO_DIR $NAME origin
 	if [ $? -ne 0 ]; then
-            error $REPO_DIR "git-mirror.sh failed with module ${NAME}" "Can not fetch from ${MASTER_REPO}" 
+	    # format mail body
+	    body="Can not fetch from ${MASTER_REPO}\n\n------------\n FAILED COMMANDS:\n"
+	    for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
+            error $REPO_DIR "git-mirror.sh failed to merge remote with module ${NAME}" "$body"
 	fi
     else
         msg "mirroring ${NAME} for the first time"
@@ -145,16 +150,22 @@ function mirror_repo () {
         STATUS_OUT=$(git status)
         popd > /dev/null
 	# format mail body
-	body="STATUS (in $REPO_DIR):\n$STATUS_OUT \n\n------------\n FAILED COMMANDS:\n"
+	body="STATUS in ${REPO_DIR}:\n${STATUS_OUT} \n\n------------\n FAILED COMMANDS:\n"
 	for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
-        error $REPO_DIR "git-mirror.sh failed on with module ${NAME}" $body
-    else
-        run ${REPO_DIR} git push --tags local_master
-        push_all_branches $REPO_DIR $NAME local_master origin 
-	
-        # success, remove previous check file if any
-        rm -f $REPO_DIR/$NOTIFIED_FILE
+        error $REPO_DIR "git-mirror.sh failed on with module ${NAME}" "$body"
+	return
+    fi
+    run ${REPO_DIR} git push --tags local_master
+    push_all_branches $REPO_DIR $NAME local_master origin 
+    if [ $? -ne 0 ]; then
+	# format mail body
+	body="FAILED COMMANDS:\n"
+	for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
+        error $REPO_DIR "git-mirror.sh failed to push back with module ${NAME}" "$body"
+	return
     fi
+    # success, remove previous check file if any
+    rm -f $REPO_DIR/$NOTIFIED_FILE
 }
 
 function usage () {
@@ -186,7 +197,7 @@ if [ -f $RUNNING_FILE ] ; then
     exit 1
 fi
 
-trap failure ERR INT
+trap failure INT
 
 shift $((OPTIND-1))
 date > $RUNNING_FILE
-- 
2.43.0