X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=scripts%2Fgit-mirror.sh;h=3f05125dfb8410fc4e0ba803ef78e58b953b08c2;hb=8784cfd2db996cd5b065d40d4eba282557056602;hp=7768b48583eb09bfe3c90733014ab5e582c91877;hpb=7e4fea9d6203a056790e07d1e0200c5712dd6be8;p=infrastructure.git

diff --git a/scripts/git-mirror.sh b/scripts/git-mirror.sh
index 7768b48..3f05125 100755
--- a/scripts/git-mirror.sh
+++ b/scripts/git-mirror.sh
@@ -2,45 +2,63 @@
 COMMAND=$(basename $0)
 
 # for sending emails (-a option)
-ADMINS="Thierry.Parmentelat@inria.fr baris@metin.org"
+ADMINS="Thierry.Parmentelat@inria.fr"
 # the other end of the mirror (-r option)
 REMOTE_GIT="git://git.planet-lab.org"
-# -q option
+# options
 QUIET=
+VERBOSE=
+FORCE=
 
 # local path to the reference (bare) repos
 MASTER_GIT="/git"
 # local path to the working repos; can be trashed if needed
 LOCAL_MIRROR_DIR="/git-mirror"
 # a file in each repo to avoid too many notifications
-NOTIFIED_FILE="NOTIFIED_ADMINS"
+NOTIFIED_FILE="NOTIFIED"
 # global stamp to avoid multiple instances of this script
-RUNNING_FILE=$LOCAL_MIRROR_DIR/RUNNING_MIRROR
+LOCKFILE=$LOCAL_MIRROR_DIR/LOCK
 # global list - errors to report
 FAILED_CMDS=()
 
+function verbose () {
+    [ -n "$VERBOSE" ] && echo "--------------------" "$@"
+}
+
 function msg () {
     [ -n "$QUIET" ] || echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" "$@"
 }
 
 function failure () {
-    msg "Received signal - cleaning up $RUNNING_FILE and aborting"
-    rm -f $RUNNING_FILE
+    msg "Received signal - cleaning up $LOCKFILE and aborting"
+    rm -f $LOCKFILE
     exit 1
 }
 
-function error () {
-    local REPO=$1; shift
+function notify () {
+    local REPO_DIR=$1; shift
     local SUBJECT=$1; shift
     local BODY=$1; shift
-
-    # already notified ? 
-    [ -f $REPO/$NOTIFIED_FILE ] && return
+    verbose "notify" $SUBJECT
+
+    # already notified within a half day ?
+    GRACE=$((60*12))
+    is_recent=$(find $REPO_DIR/$NOTIFIED_FILE -mmin -$GRACE 2> /dev/null)
+    # not there or less than 12 our old: don't notify
+    if [ -n "$is_recent" ] ; then
+	verbose "skipping recent notification -- $SUBJECT"
+	return
+    fi
 
     for admin in $ADMINS; do
         echo -e "$BODY" | mail -s "$SUBJECT" $admin
     done
-    touch $REPO/$NOTIFIED_FILE
+    touch $REPO_DIR/$NOTIFIED_FILE
+}
+
+function clear_notify () {
+    local REPO_DIR=$1; shift
+    rm -f $REPO_DIR/$NOTIFIED_FILE
 }
 
 # run directory command
@@ -49,20 +67,26 @@ function error () {
 function run () {
     local IGNORE=""
     case "$1" in -i) IGNORE=true ; shift ;; esac
-    local REPO=$1; shift
+    local REPO_DIR=$1; shift
     local COMMAND="$@"
 
-    pushd ${REPO} > /dev/null
+    pushd ${REPO_DIR} > /dev/null
     # quiet mode: record stdout and err for the mail
     ok=true
     if [ -n "$QUIET" ] ; then
 	OUTPUT=$($COMMAND 2>&1) || ok=
-	TORECORD="[$REPO] $OUTPUT"
+	TORECORD="[$REPO_DIR] $COMMAND\n$OUTPUT"
     else
-	echo "[$REPO] Running $COMMAND"
-	$COMMAND || ok=
-	TORECORD="$COMMAND"
+	if [ -z "$IGNORE" ] ; then
+	    echo "[$REPO_DIR] Running $COMMAND"
+	    $COMMAND || ok=
+	    TORECORD="$COMMAND"
+	else
+	    echo "[$REPO_DIR] Running $COMMAND [ignored]"
+	    $COMMAND >& /dev/null
+	fi
     fi
+    
     # failed ?
     if [ -z "$ok" ]; then
 	# let's record the failure unless ignore is set
@@ -70,6 +94,7 @@ function run () {
             FAILED_CMDS=("${FAILED_CMDS[@]}" "$TORECORD")
 	fi
     fi
+    verbose "after run with $COMMAND" "ok=$ok" "#failed=${#FAILED_CMDS[@]}"
     popd > /dev/null
 }
 
@@ -80,12 +105,13 @@ function merge_all_branches () {
 
     pushd $REPO_DIR > /dev/null
     BRANCHES=$(git branch -r | grep $REMOTE | grep -v HEAD | sed "s/.*\///g" | grep -v master)
+    HAS_MASTER=$(git branch -l | grep master)
     popd > /dev/null
 
-    run ${REPO_DIR} git checkout master
-    run ${REPO_DIR} git merge --ff $REMOTE/master
+    [ -n "$HAS_MASTER" ] && run ${REPO_DIR} git checkout master
+    [ -n "$HAS_MASTER" ] && run ${REPO_DIR} git merge --ff $REMOTE/master
     for BRANCH in $BRANCHES ; do
-        run -i  ${REPO_DIR} git branch $BRANCH $REMOTE/$BRANCH
+        run -i ${REPO_DIR} git branch $BRANCH $REMOTE/$BRANCH
         run ${REPO_DIR} git checkout $BRANCH
         run ${REPO_DIR} git merge --ff $REMOTE/$BRANCH
     done
@@ -99,9 +125,10 @@ function push_all_branches () {
 
     pushd $REPO_DIR > /dev/null
     BRANCHES=$(git branch -r | grep $PUSH_FROM | grep -v HEAD | sed "s/.*\///g" | grep -v master)
+    HAS_MASTER=$(git branch -l | grep master)
     popd > /dev/null
 
-    run ${REPO_DIR} git push $PUSH_TO master:master
+    [ -n "$HAS_MASTER" ] && run ${REPO_DIR} git push $PUSH_TO master:master
     for BRANCH in $BRANCHES ; do
         run ${REPO_DIR} git push $PUSH_TO $BRANCH:$BRANCH
     done
@@ -117,6 +144,12 @@ function mirror_repo () {
     REMOTE_REPO=${REMOTE_GIT}/${GIT_NAME}
     MASTER_REPO=${MASTER_GIT}/${GIT_NAME}
 
+    # if the local master is a symlink (like /git/vserver-reference.git -> sliceref.git) 
+    # then skip it
+    # we use this for either aliases (like vserver-reference and sliceimage) or
+    # for repos managed in other locations (like /git-slave) but where
+    # the symlink is needed so they get served by git-daemon
+    [ -h ${MASTER_REPO} ] && return
 
     # if there is no remote repository it may be that we only have
     # the repository locally and don't need to mirror
@@ -127,8 +160,12 @@ function mirror_repo () {
 	run ${REPO_DIR} git fetch origin --tags
 	run ${REPO_DIR} git fetch origin
 	merge_all_branches $REPO_DIR $NAME origin
-	if [ $? -ne 0 ]; then
-            error $REPO_DIR "git-mirror.sh failed with module ${NAME}" "Can not fetch from ${MASTER_REPO}" 
+	if [ -n "$FAILED_CMDS" ]; then
+	    # format mail body
+	    body="Can not fetch from ${MASTER_REPO}\n\n------------\n FAILED COMMANDS:\n"
+	    for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
+            notify $REPO_DIR "git-mirror.sh failed to merge remote with module ${NAME}" "$body"
+	    return
 	fi
     else
         msg "mirroring ${NAME} for the first time"
@@ -145,50 +182,72 @@ function mirror_repo () {
         STATUS_OUT=$(git status)
         popd > /dev/null
 	# format mail body
-	body="STATUS (in $REPO_DIR):\n$STATUS_OUT \n\n------------\n FAILED COMMANDS:\n"
+	body="STATUS in ${REPO_DIR}:\n${STATUS_OUT} \n\n------------\n FAILED COMMANDS:\n"
 	for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
-        error $REPO_DIR "git-mirror.sh failed on with module ${NAME}" $body
-    else
-        run ${REPO_DIR} git push --tags local_master
-        push_all_branches $REPO_DIR $NAME local_master origin 
-	
-        # success, remove previous check file if any
-        rm -f $REPO_DIR/$NOTIFIED_FILE
+        notify $REPO_DIR "git-mirror.sh failed on with module ${NAME}" "$body"
+	return
+    fi
+    run ${REPO_DIR} git push --tags local_master
+    push_all_branches $REPO_DIR $NAME local_master origin 
+    if [ -n "$FAILED_CMDS" ]; then
+	# format mail body
+	body="FAILED COMMANDS:\n"
+	for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
+        notify $REPO_DIR "git-mirror.sh failed to push back with module ${NAME}" "$body"
+	return
     fi
+    # success, remove previous check file if any
+    clear_notify $REPO_DIR
+    # touch a stamp so it's easier to figure out where/if things get stuck
+    touch ${REPO_DIR}/MIRRORED.stamp
 }
 
 function usage () {
-    echo "Usage $COMMAND [-a admin-mails] [-r remote-git-url] [-q] REPONAME*"
+    echo "Usage $COMMAND [options] REPONAME*"
+    echo "  [-a admin-mails] : provide space-separated admins emails"
+    echo "  [-r remote-git-url] : e.g. -r git://git.onelab.eu/"
+    echo "  [-q] quiet mode for running under cron"
+    echo "  [-v] verbose mode"
+    echo "  [-f] force mode, runs even if the lock file is present"
+    echo "       see also manage-git-mirror.sh"
     exit 1
 }
 
-while getopts "a:r:qh" opt; do
+while getopts "a:r:s:qvfh" opt; do
   case $opt in
       a) ADMINS=$OPTARG ;;
       r) REMOTE_GIT=$OPTARG ;;
       q) QUIET=true ;;
+      v) VERBOSE=true ;;
+      f) FORCE=true ;;
       h) usage ;;
       \?) echo "Invalid option: -$opt" >&2 ;;
   esac
 done
+shift $((OPTIND-1))
 
-# is the stamp older than an hour ? 
-# in minutes
-GRACE=60
-is_old=$(find $RUNNING_FILE -mmin +$GRACE 2> /dev/null)
-if [ -n "$is_old" ] ; then
-    msg "$RUNNING_FILE is older than $GRACE minutes - removing"
-    rm -f $RUNNING_FILE
-fi
+# skip this if force is set
+# the natural usage of force is, manage-git-mirror.sh touches the lock file
+# which prevents the cron jobs from running during an hour
+# then you can use with -f which leaves the lock file as it is
+if [ -z "$FORCE" ] ; then
+    # is the stamp older than an hour ? 
+    GRACE=60
+    is_old=$(find $LOCKFILE -mmin +$GRACE 2> /dev/null)
+    if [ -n "$is_old" ] ; then
+	msg "$LOCKFILE is older than $GRACE minutes - removing"
+	rm -f $LOCKFILE
+    fi
 
-if [ -f $RUNNING_FILE ] ; then
-    msg "Found $RUNNING_FILE - another git-mirror seems to be running. Aborting... " 
-    exit 1
+    if [ -f $LOCKFILE ] ; then
+	msg "Found $LOCKFILE - another git-mirror seems to be running. Aborting... " 
+	exit 1
+    fi
 fi
 
-trap failure ERR INT
+trap failure INT
 
-shift $((OPTIND-1))
-date > $RUNNING_FILE
+# if force is set we leave the lock file as is
+[ -z "$FORCE" ] && date > $LOCKFILE
 for gitrepo in "$@"; do mirror_repo $gitrepo ; done
-rm -f $RUNNING_FILE
+[ -z "$FORCE" ] && rm -f $LOCKFILE