X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=scripts%2Fgit-mirror.sh;h=3f05125dfb8410fc4e0ba803ef78e58b953b08c2;hb=HEAD;hp=434b7e8d565bfc92c30f47bd3c7ae7a023b2feb9;hpb=a88407b43212fc319e5a1b87cdfbec5340ac52b6;p=infrastructure.git

diff --git a/scripts/git-mirror.sh b/scripts/git-mirror.sh
old mode 100644
new mode 100755
index 434b7e8..3f05125
--- a/scripts/git-mirror.sh
+++ b/scripts/git-mirror.sh
@@ -1,40 +1,253 @@
 #!/bin/bash
+COMMAND=$(basename $0)
 
-MIRROR="git://git.planet-lab.org"
-LOCAL="/git"
-
-function mirror () {
-    for arg in "$@" ; do
-        REPO=${arg}
-        REPO_NAME=${REPO}.git
-        MIRROR_REPO=${MIRROR}/${REPO_NAME}
-        LOCAL_REPO=${LOCAL}/${REPO_NAME}
-
-        if [ -d ${REPO} ]
-        then
-	    echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx pulling from ${REPO_NAME}"
-            pushd ${REPO}
-            git pull --tags
-            git pull
-            popd
-        else
-            echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx mirroring in ${REPO_NAME} for the first time"
-            git clone ${MIRROR_REPO}
-            pushd ${REPO}
-            git remote add local_master ${LOCAL_REPO}
-            popd
-        fi
-
-        echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx pushing ${REPO_NAME} to local master"
-        pushd ${REPO}
-        git push local_master
-        git push --tags local_master
-        popd
+# for sending emails (-a option)
+ADMINS="Thierry.Parmentelat@inria.fr"
+# the other end of the mirror (-r option)
+REMOTE_GIT="git://git.planet-lab.org"
+# options
+QUIET=
+VERBOSE=
+FORCE=
+
+# local path to the reference (bare) repos
+MASTER_GIT="/git"
+# local path to the working repos; can be trashed if needed
+LOCAL_MIRROR_DIR="/git-mirror"
+# a file in each repo to avoid too many notifications
+NOTIFIED_FILE="NOTIFIED"
+# global stamp to avoid multiple instances of this script
+LOCKFILE=$LOCAL_MIRROR_DIR/LOCK
+# global list - errors to report
+FAILED_CMDS=()
+
+function verbose () {
+    [ -n "$VERBOSE" ] && echo "--------------------" "$@"
+}
+
+function msg () {
+    [ -n "$QUIET" ] || echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" "$@"
+}
+
+function failure () {
+    msg "Received signal - cleaning up $LOCKFILE and aborting"
+    rm -f $LOCKFILE
+    exit 1
+}
+
+function notify () {
+    local REPO_DIR=$1; shift
+    local SUBJECT=$1; shift
+    local BODY=$1; shift
+    verbose "notify" $SUBJECT
+
+    # already notified within a half day ?
+    GRACE=$((60*12))
+    is_recent=$(find $REPO_DIR/$NOTIFIED_FILE -mmin -$GRACE 2> /dev/null)
+    # not there or less than 12 our old: don't notify
+    if [ -n "$is_recent" ] ; then
+	verbose "skipping recent notification -- $SUBJECT"
+	return
+    fi
+
+    for admin in $ADMINS; do
+        echo -e "$BODY" | mail -s "$SUBJECT" $admin
+    done
+    touch $REPO_DIR/$NOTIFIED_FILE
+}
+
+function clear_notify () {
+    local REPO_DIR=$1; shift
+    rm -f $REPO_DIR/$NOTIFIED_FILE
+}
+
+# run directory command
+#   -or- to ignore errors
+# run -i directory "command"
+function run () {
+    local IGNORE=""
+    case "$1" in -i) IGNORE=true ; shift ;; esac
+    local REPO_DIR=$1; shift
+    local COMMAND="$@"
+
+    pushd ${REPO_DIR} > /dev/null
+    # quiet mode: record stdout and err for the mail
+    ok=true
+    if [ -n "$QUIET" ] ; then
+	OUTPUT=$($COMMAND 2>&1) || ok=
+	TORECORD="[$REPO_DIR] $COMMAND\n$OUTPUT"
+    else
+	if [ -z "$IGNORE" ] ; then
+	    echo "[$REPO_DIR] Running $COMMAND"
+	    $COMMAND || ok=
+	    TORECORD="$COMMAND"
+	else
+	    echo "[$REPO_DIR] Running $COMMAND [ignored]"
+	    $COMMAND >& /dev/null
+	fi
+    fi
+    
+    # failed ?
+    if [ -z "$ok" ]; then
+	# let's record the failure unless ignore is set
+	if [ -z "$IGNORE" ] ; then
+            FAILED_CMDS=("${FAILED_CMDS[@]}" "$TORECORD")
+	fi
+    fi
+    verbose "after run with $COMMAND" "ok=$ok" "#failed=${#FAILED_CMDS[@]}"
+    popd > /dev/null
+}
+
+function merge_all_branches () {
+    local REPO_DIR=$1; shift
+    local NAME=$1; shift
+    local REMOTE=$1; shift
+
+    pushd $REPO_DIR > /dev/null
+    BRANCHES=$(git branch -r | grep $REMOTE | grep -v HEAD | sed "s/.*\///g" | grep -v master)
+    HAS_MASTER=$(git branch -l | grep master)
+    popd > /dev/null
+
+    [ -n "$HAS_MASTER" ] && run ${REPO_DIR} git checkout master
+    [ -n "$HAS_MASTER" ] && run ${REPO_DIR} git merge --ff $REMOTE/master
+    for BRANCH in $BRANCHES ; do
+        run -i ${REPO_DIR} git branch $BRANCH $REMOTE/$BRANCH
+        run ${REPO_DIR} git checkout $BRANCH
+        run ${REPO_DIR} git merge --ff $REMOTE/$BRANCH
     done
 }
 
-args="$@"
-[[ -z "$args" ]] && args=$(ls /svn)
+function push_all_branches () {
+    local REPO_DIR=$1; shift
+    local NAME=$1; shift
+    local PUSH_TO=$1; shift
+    local PUSH_FROM=$1; shift
+
+    pushd $REPO_DIR > /dev/null
+    BRANCHES=$(git branch -r | grep $PUSH_FROM | grep -v HEAD | sed "s/.*\///g" | grep -v master)
+    HAS_MASTER=$(git branch -l | grep master)
+    popd > /dev/null
+
+    [ -n "$HAS_MASTER" ] && run ${REPO_DIR} git push $PUSH_TO master:master
+    for BRANCH in $BRANCHES ; do
+        run ${REPO_DIR} git push $PUSH_TO $BRANCH:$BRANCH
+    done
+}
+
+function mirror_repo () {
+    local repo=$1; shift
+    FAILED_CMDS=()   # reset previous failure if any
+
+    NAME=$(basename ${repo} ".git")
+    GIT_NAME=${NAME}.git
+    REPO_DIR=${LOCAL_MIRROR_DIR}/${NAME}
+    REMOTE_REPO=${REMOTE_GIT}/${GIT_NAME}
+    MASTER_REPO=${MASTER_GIT}/${GIT_NAME}
+
+    # if the local master is a symlink (like /git/vserver-reference.git -> sliceref.git) 
+    # then skip it
+    # we use this for either aliases (like vserver-reference and sliceimage) or
+    # for repos managed in other locations (like /git-slave) but where
+    # the symlink is needed so they get served by git-daemon
+    [ -h ${MASTER_REPO} ] && return
+
+    # if there is no remote repository it may be that we only have
+    # the repository locally and don't need to mirror
+    git ls-remote $REMOTE_REPO &> /dev/null || return
+
+    if [ -d ${REPO_DIR} ] ; then
+	msg "pulling from ${NAME}"
+	run ${REPO_DIR} git fetch origin --tags
+	run ${REPO_DIR} git fetch origin
+	merge_all_branches $REPO_DIR $NAME origin
+	if [ -n "$FAILED_CMDS" ]; then
+	    # format mail body
+	    body="Can not fetch from ${MASTER_REPO}\n\n------------\n FAILED COMMANDS:\n"
+	    for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
+            notify $REPO_DIR "git-mirror.sh failed to merge remote with module ${NAME}" "$body"
+	    return
+	fi
+    else
+        msg "mirroring ${NAME} for the first time"
+        run ${LOCAL_MIRROR_DIR} git clone ${REMOTE_REPO}
+        run ${REPO_DIR} git remote add local_master ${MASTER_REPO}
+    fi
+
+    msg "pushing ${NAME} to local master"
+    run ${REPO_DIR} git fetch local_master --tags
+    run ${REPO_DIR} git fetch local_master
+    merge_all_branches $REPO_DIR $NAME local_master
+    if [ -n "$FAILED_CMDS" ]; then
+        pushd ${REPO_DIR} > /dev/null
+        STATUS_OUT=$(git status)
+        popd > /dev/null
+	# format mail body
+	body="STATUS in ${REPO_DIR}:\n${STATUS_OUT} \n\n------------\n FAILED COMMANDS:\n"
+	for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
+        notify $REPO_DIR "git-mirror.sh failed on with module ${NAME}" "$body"
+	return
+    fi
+    run ${REPO_DIR} git push --tags local_master
+    push_all_branches $REPO_DIR $NAME local_master origin 
+    if [ -n "$FAILED_CMDS" ]; then
+	# format mail body
+	body="FAILED COMMANDS:\n"
+	for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
+        notify $REPO_DIR "git-mirror.sh failed to push back with module ${NAME}" "$body"
+	return
+    fi
+    # success, remove previous check file if any
+    clear_notify $REPO_DIR
+    # touch a stamp so it's easier to figure out where/if things get stuck
+    touch ${REPO_DIR}/MIRRORED.stamp
+}
+
+function usage () {
+    echo "Usage $COMMAND [options] REPONAME*"
+    echo "  [-a admin-mails] : provide space-separated admins emails"
+    echo "  [-r remote-git-url] : e.g. -r git://git.onelab.eu/"
+    echo "  [-q] quiet mode for running under cron"
+    echo "  [-v] verbose mode"
+    echo "  [-f] force mode, runs even if the lock file is present"
+    echo "       see also manage-git-mirror.sh"
+    exit 1
+}
+
+while getopts "a:r:s:qvfh" opt; do
+  case $opt in
+      a) ADMINS=$OPTARG ;;
+      r) REMOTE_GIT=$OPTARG ;;
+      q) QUIET=true ;;
+      v) VERBOSE=true ;;
+      f) FORCE=true ;;
+      h) usage ;;
+      \?) echo "Invalid option: -$opt" >&2 ;;
+  esac
+done
+shift $((OPTIND-1))
+
+# skip this if force is set
+# the natural usage of force is, manage-git-mirror.sh touches the lock file
+# which prevents the cron jobs from running during an hour
+# then you can use with -f which leaves the lock file as it is
+if [ -z "$FORCE" ] ; then
+    # is the stamp older than an hour ? 
+    GRACE=60
+    is_old=$(find $LOCKFILE -mmin +$GRACE 2> /dev/null)
+    if [ -n "$is_old" ] ; then
+	msg "$LOCKFILE is older than $GRACE minutes - removing"
+	rm -f $LOCKFILE
+    fi
+
+    if [ -f $LOCKFILE ] ; then
+	msg "Found $LOCKFILE - another git-mirror seems to be running. Aborting... " 
+	exit 1
+    fi
+fi
 
-mirror $args
+trap failure INT
 
+# if force is set we leave the lock file as is
+[ -z "$FORCE" ] && date > $LOCKFILE
+for gitrepo in "$@"; do mirror_repo $gitrepo ; done
+[ -z "$FORCE" ] && rm -f $LOCKFILE