X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=scripts%2Fgit-mirror.sh;h=3f05125dfb8410fc4e0ba803ef78e58b953b08c2;hb=HEAD;hp=8ebae5bc4cc2fb3a091968d19243d2ae589e647f;hpb=e39f99e87e04b9611940aefaac24ecff1665861c;p=infrastructure.git

diff --git a/scripts/git-mirror.sh b/scripts/git-mirror.sh
index 8ebae5b..3f05125 100755
--- a/scripts/git-mirror.sh
+++ b/scripts/git-mirror.sh
@@ -1,158 +1,253 @@
 #!/bin/bash
-
-ADMINS="Talip-Baris.Metin@inria.fr"
-MIRROR_GIT="git://git.planet-lab.org"
+COMMAND=$(basename $0)
+
+# for sending emails (-a option)
+ADMINS="Thierry.Parmentelat@inria.fr"
+# the other end of the mirror (-r option)
+REMOTE_GIT="git://git.planet-lab.org"
+# options
+QUIET=
+VERBOSE=
+FORCE=
+
+# local path to the reference (bare) repos
 MASTER_GIT="/git"
+# local path to the working repos; can be trashed if needed
 LOCAL_MIRROR_DIR="/git-mirror"
-QUIET=0
-FAILED=0
-FAILED_CMDS=""
+# a file in each repo to avoid too many notifications
+NOTIFIED_FILE="NOTIFIED"
+# global stamp to avoid multiple instances of this script
+LOCKFILE=$LOCAL_MIRROR_DIR/LOCK
+# global list - errors to report
+FAILED_CMDS=()
+
+function verbose () {
+    [ -n "$VERBOSE" ] && echo "--------------------" "$@"
+}
 
 function msg () {
-    if [ $QUIET -eq 0 ]
-    then
-        echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx $@"
-    fi
+    [ -n "$QUIET" ] || echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" "$@"
 }
 
-function error () {
-    SUBJECT=$1
-    MSG=$2
-    for admin in $ADMINS
-    do
-        echo -e "$MSG" | mail -s "$SUBJECT" $admin
+function failure () {
+    msg "Received signal - cleaning up $LOCKFILE and aborting"
+    rm -f $LOCKFILE
+    exit 1
+}
+
+function notify () {
+    local REPO_DIR=$1; shift
+    local SUBJECT=$1; shift
+    local BODY=$1; shift
+    verbose "notify" $SUBJECT
+
+    # already notified within a half day ?
+    GRACE=$((60*12))
+    is_recent=$(find $REPO_DIR/$NOTIFIED_FILE -mmin -$GRACE 2> /dev/null)
+    # not there or less than 12 our old: don't notify
+    if [ -n "$is_recent" ] ; then
+	verbose "skipping recent notification -- $SUBJECT"
+	return
+    fi
+
+    for admin in $ADMINS; do
+        echo -e "$BODY" | mail -s "$SUBJECT" $admin
     done
+    touch $REPO_DIR/$NOTIFIED_FILE
 }
 
+function clear_notify () {
+    local REPO_DIR=$1; shift
+    rm -f $REPO_DIR/$NOTIFIED_FILE
+}
+
+# run directory command
+#   -or- to ignore errors
+# run -i directory "command"
 function run () {
-    if [ $QUIET -eq 1 ]
-    then
-        COMMAND="$1 &> /dev/null"
+    local IGNORE=""
+    case "$1" in -i) IGNORE=true ; shift ;; esac
+    local REPO_DIR=$1; shift
+    local COMMAND="$@"
+
+    pushd ${REPO_DIR} > /dev/null
+    # quiet mode: record stdout and err for the mail
+    ok=true
+    if [ -n "$QUIET" ] ; then
+	OUTPUT=$($COMMAND 2>&1) || ok=
+	TORECORD="[$REPO_DIR] $COMMAND\n$OUTPUT"
     else
-        COMMAND="$1"
-        msg $COMMAND
+	if [ -z "$IGNORE" ] ; then
+	    echo "[$REPO_DIR] Running $COMMAND"
+	    $COMMAND || ok=
+	    TORECORD="$COMMAND"
+	else
+	    echo "[$REPO_DIR] Running $COMMAND [ignored]"
+	    $COMMAND >& /dev/null
+	fi
     fi
-    REPO=$2
-
-    pushd ${REPO} > /dev/null
-    eval $COMMAND
-    if [ $? -ne 0 ]
-    then
-        FAILED=1
-        FAILED_CMDS="$FAILED_CMDS\n$COMMAND" 
+    
+    # failed ?
+    if [ -z "$ok" ]; then
+	# let's record the failure unless ignore is set
+	if [ -z "$IGNORE" ] ; then
+            FAILED_CMDS=("${FAILED_CMDS[@]}" "$TORECORD")
+	fi
     fi
+    verbose "after run with $COMMAND" "ok=$ok" "#failed=${#FAILED_CMDS[@]}"
     popd > /dev/null
 }
 
-function run_ignore_errors () {
-    OLD_FAILED=$FAILED
-    OLD_FAILED_CMDS=$FAILED_CMDS
-    run "$1" "$2"
-    FAILED=$OLD_FAILED
-    FAILED_CMDS=$OLD_FAILED_CMDS
-}
-
 function merge_all_branches () {
-    NAME=$1
-    REMOTE=$2
-    REPO_DIR=$3
+    local REPO_DIR=$1; shift
+    local NAME=$1; shift
+    local REMOTE=$1; shift
 
     pushd $REPO_DIR > /dev/null
     BRANCHES=$(git branch -r | grep $REMOTE | grep -v HEAD | sed "s/.*\///g" | grep -v master)
+    HAS_MASTER=$(git branch -l | grep master)
     popd > /dev/null
 
-    run "git checkout master" ${REPO_DIR}
-    run "git merge --ff $REMOTE/master" ${REPO_DIR}
+    [ -n "$HAS_MASTER" ] && run ${REPO_DIR} git checkout master
+    [ -n "$HAS_MASTER" ] && run ${REPO_DIR} git merge --ff $REMOTE/master
     for BRANCH in $BRANCHES ; do
-        run_ignore_errors "git branch $BRANCH $REMOTE/$BRANCH" ${REPO_DIR}
-        run "git checkout $BRANCH" ${REPO_DIR}
-        run "git merge --ff $REMOTE/$BRANCH" ${REPO_DIR}
+        run -i ${REPO_DIR} git branch $BRANCH $REMOTE/$BRANCH
+        run ${REPO_DIR} git checkout $BRANCH
+        run ${REPO_DIR} git merge --ff $REMOTE/$BRANCH
     done
 }
 
 function push_all_branches () {
-    NAME=$1
-    REMOTE=$2
-    REPO_DIR=$3
+    local REPO_DIR=$1; shift
+    local NAME=$1; shift
+    local PUSH_TO=$1; shift
+    local PUSH_FROM=$1; shift
 
     pushd $REPO_DIR > /dev/null
-    BRANCHES=$(git branch -r | grep $REMOTE | grep -v HEAD | sed "s/.*\///g" | grep -v master)
+    BRANCHES=$(git branch -r | grep $PUSH_FROM | grep -v HEAD | sed "s/.*\///g" | grep -v master)
+    HAS_MASTER=$(git branch -l | grep master)
     popd > /dev/null
 
-    run "git push $REMOTE master:master" ${REPO_DIR}
+    [ -n "$HAS_MASTER" ] && run ${REPO_DIR} git push $PUSH_TO master:master
     for BRANCH in $BRANCHES ; do
-        run "git push $REMOTE $BRANCH:$BRANCH" ${REPO_DIR}
+        run ${REPO_DIR} git push $PUSH_TO $BRANCH:$BRANCH
     done
 }
 
-function mirror () {
-    for arg in "$@" ; do
-        FAILED=0   # reset previous failure if any
-
-        NAME=$(basename ${arg} | sed s/.git$//g)
-        GIT_NAME=${NAME}.git
-        REPO_DIR=${LOCAL_MIRROR_DIR}/${NAME}
-        MIRROR_REPO=${MIRROR_GIT}/${GIT_NAME}
-        MASTER_REPO=${MASTER_GIT}/${GIT_NAME}
-
-
-        # if there is no remote repository it may be that we only have
-        # the repository locally and don't need to mirror
-        git ls-remote $MIRROR_REPO &> /dev/null
-        if [ $? -eq 0 ]
-        then
-            if [ -d ${REPO_DIR} ]
-            then
-	        msg "pulling from ${NAME}"
-                run "git fetch origin --tags" ${REPO_DIR}
-	        run "git fetch origin" ${REPO_DIR}
-                merge_all_branches $NAME origin $REPO_DIR
-                if [ $? -ne 0 ]
-                then
-                    error "Can not fetch from ${MASTER_REPO}"
-                fi
-            else
-                msg "mirroring ${NAME} for the first time"
-                run "git clone ${MIRROR_REPO}" ${LOCAL_MIRROR_DIR}
-                run "git remote add local_master ${MASTER_REPO}" ${REPO_DIR}
-            fi
-
-            msg "pushing ${NAME} to local master"
-            run "git fetch local_master --tags" ${REPO_DIR}
-            run "git fetch local_master" ${REPO_DIR}
-            merge_all_branches $NAME local_master $REPO_DIR
-            if [ $FAILED -ne 0 ]
-            then
-                pushd ${REPO_DIR} > /dev/null
-                STATUS_OUT=$(git status)
-                popd > /dev/null                
-                error "git-mirror.sh failed on ${MIRROR_REPO}" "STATUS:\n$STATUS_OUT \n\n------------\n FAILED COMMANDS:\n$FAILED_CMDS"
-            else
-                run "git push --tags local_master" ${REPO_DIR}
-                push_all_branches $NAME local_master $REPO_DIR
-            fi
-        fi
-    done
+function mirror_repo () {
+    local repo=$1; shift
+    FAILED_CMDS=()   # reset previous failure if any
+
+    NAME=$(basename ${repo} ".git")
+    GIT_NAME=${NAME}.git
+    REPO_DIR=${LOCAL_MIRROR_DIR}/${NAME}
+    REMOTE_REPO=${REMOTE_GIT}/${GIT_NAME}
+    MASTER_REPO=${MASTER_GIT}/${GIT_NAME}
+
+    # if the local master is a symlink (like /git/vserver-reference.git -> sliceref.git) 
+    # then skip it
+    # we use this for either aliases (like vserver-reference and sliceimage) or
+    # for repos managed in other locations (like /git-slave) but where
+    # the symlink is needed so they get served by git-daemon
+    [ -h ${MASTER_REPO} ] && return
+
+    # if there is no remote repository it may be that we only have
+    # the repository locally and don't need to mirror
+    git ls-remote $REMOTE_REPO &> /dev/null || return
+
+    if [ -d ${REPO_DIR} ] ; then
+	msg "pulling from ${NAME}"
+	run ${REPO_DIR} git fetch origin --tags
+	run ${REPO_DIR} git fetch origin
+	merge_all_branches $REPO_DIR $NAME origin
+	if [ -n "$FAILED_CMDS" ]; then
+	    # format mail body
+	    body="Can not fetch from ${MASTER_REPO}\n\n------------\n FAILED COMMANDS:\n"
+	    for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
+            notify $REPO_DIR "git-mirror.sh failed to merge remote with module ${NAME}" "$body"
+	    return
+	fi
+    else
+        msg "mirroring ${NAME} for the first time"
+        run ${LOCAL_MIRROR_DIR} git clone ${REMOTE_REPO}
+        run ${REPO_DIR} git remote add local_master ${MASTER_REPO}
+    fi
+
+    msg "pushing ${NAME} to local master"
+    run ${REPO_DIR} git fetch local_master --tags
+    run ${REPO_DIR} git fetch local_master
+    merge_all_branches $REPO_DIR $NAME local_master
+    if [ -n "$FAILED_CMDS" ]; then
+        pushd ${REPO_DIR} > /dev/null
+        STATUS_OUT=$(git status)
+        popd > /dev/null
+	# format mail body
+	body="STATUS in ${REPO_DIR}:\n${STATUS_OUT} \n\n------------\n FAILED COMMANDS:\n"
+	for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
+        notify $REPO_DIR "git-mirror.sh failed on with module ${NAME}" "$body"
+	return
+    fi
+    run ${REPO_DIR} git push --tags local_master
+    push_all_branches $REPO_DIR $NAME local_master origin 
+    if [ -n "$FAILED_CMDS" ]; then
+	# format mail body
+	body="FAILED COMMANDS:\n"
+	for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
+        notify $REPO_DIR "git-mirror.sh failed to push back with module ${NAME}" "$body"
+	return
+    fi
+    # success, remove previous check file if any
+    clear_notify $REPO_DIR
+    # touch a stamp so it's easier to figure out where/if things get stuck
+    touch ${REPO_DIR}/MIRRORED.stamp
 }
 
+function usage () {
+    echo "Usage $COMMAND [options] REPONAME*"
+    echo "  [-a admin-mails] : provide space-separated admins emails"
+    echo "  [-r remote-git-url] : e.g. -r git://git.onelab.eu/"
+    echo "  [-q] quiet mode for running under cron"
+    echo "  [-v] verbose mode"
+    echo "  [-f] force mode, runs even if the lock file is present"
+    echo "       see also manage-git-mirror.sh"
+    exit 1
+}
 
-while getopts ":hq" opt
-do
+while getopts "a:r:s:qvfh" opt; do
   case $opt in
-      q)
-          QUIET=1
-          break
-          ;;
-      h)
-          echo "USAGE: $0 [-q] REPONAME*"
-          exit 1
-          ;;
-      \?)
-          echo "Invalid option: -$OPTARG" >&2
-          ;;
+      a) ADMINS=$OPTARG ;;
+      r) REMOTE_GIT=$OPTARG ;;
+      q) QUIET=true ;;
+      v) VERBOSE=true ;;
+      f) FORCE=true ;;
+      h) usage ;;
+      \?) echo "Invalid option: -$opt" >&2 ;;
   esac
 done
-
 shift $((OPTIND-1))
-mirror $@
 
+# skip this if force is set
+# the natural usage of force is, manage-git-mirror.sh touches the lock file
+# which prevents the cron jobs from running during an hour
+# then you can use with -f which leaves the lock file as it is
+if [ -z "$FORCE" ] ; then
+    # is the stamp older than an hour ? 
+    GRACE=60
+    is_old=$(find $LOCKFILE -mmin +$GRACE 2> /dev/null)
+    if [ -n "$is_old" ] ; then
+	msg "$LOCKFILE is older than $GRACE minutes - removing"
+	rm -f $LOCKFILE
+    fi
+
+    if [ -f $LOCKFILE ] ; then
+	msg "Found $LOCKFILE - another git-mirror seems to be running. Aborting... " 
+	exit 1
+    fi
+fi
+
+trap failure INT
+
+# if force is set we leave the lock file as is
+[ -z "$FORCE" ] && date > $LOCKFILE
+for gitrepo in "$@"; do mirror_repo $gitrepo ; done
+[ -z "$FORCE" ] && rm -f $LOCKFILE