#!/bin/bash
+COMMAND=$(basename $0)
-MIRROR_GIT="git://git.planet-lab.org"
+# for sending emails (-a option)
+ADMINS="Thierry.Parmentelat@inria.fr"
+# the other end of the mirror (-r option)
+REMOTE_GIT="git://git.planet-lab.org"
+# options
+QUIET=
+VERBOSE=
+FORCE=
+
+# local path to the reference (bare) repos
MASTER_GIT="/git"
+# local path to the working repos; can be trashed if needed
LOCAL_MIRROR_DIR="/git-mirror"
-QUIET=0
+# a file in each repo to avoid too many notifications
+NOTIFIED_FILE="NOTIFIED"
+# global stamp to avoid multiple instances of this script
+LOCKFILE=$LOCAL_MIRROR_DIR/LOCK
+# global list - errors to report
+FAILED_CMDS=()
+
+function verbose () {
+ [ -n "$VERBOSE" ] && echo "--------------------" "$@"
+}
function msg () {
- if [ $QUIET -eq 0 ]
- then
- echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx $1"
+ [ -n "$QUIET" ] || echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" "$@"
+}
+
+function failure () {
+ msg "Received signal - cleaning up $LOCKFILE and aborting"
+ rm -f $LOCKFILE
+ exit 1
+}
+
+function notify () {
+ local REPO_DIR=$1; shift
+ local SUBJECT=$1; shift
+ local BODY=$1; shift
+ verbose "notify" $SUBJECT
+
+ # already notified within a half day ?
+ GRACE=$((60*12))
+ is_recent=$(find $REPO_DIR/$NOTIFIED_FILE -mmin -$GRACE 2> /dev/null)
+ # not there or less than 12 our old: don't notify
+ if [ -n "$is_recent" ] ; then
+ verbose "skipping recent notification -- $SUBJECT"
+ return
fi
+
+ for admin in $ADMINS; do
+ echo -e "$BODY" | mail -s "$SUBJECT" $admin
+ done
+ touch $REPO_DIR/$NOTIFIED_FILE
}
-function error () {
- echo "[ERROR] xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx $1"
+function clear_notify () {
+ local REPO_DIR=$1; shift
+ rm -f $REPO_DIR/$NOTIFIED_FILE
}
+# run directory command
+# -or- to ignore errors
+# run -i directory "command"
function run () {
- if [ $QUIET -eq 1 ]
- then
- COMMAND="$1 &> /dev/null"
+ local IGNORE=""
+ case "$1" in -i) IGNORE=true ; shift ;; esac
+ local REPO_DIR=$1; shift
+ local COMMAND="$@"
+
+ pushd ${REPO_DIR} > /dev/null
+ # quiet mode: record stdout and err for the mail
+ ok=true
+ if [ -n "$QUIET" ] ; then
+ OUTPUT=$($COMMAND 2>&1) || ok=
+ TORECORD="[$REPO_DIR] $COMMAND\n$OUTPUT"
else
- COMMAND="$1"
- msg $COMMAND
+ if [ -z "$IGNORE" ] ; then
+ echo "[$REPO_DIR] Running $COMMAND"
+ $COMMAND || ok=
+ TORECORD="$COMMAND"
+ else
+ echo "[$REPO_DIR] Running $COMMAND [ignored]"
+ $COMMAND >& /dev/null
+ fi
+ fi
+
+ # failed ?
+ if [ -z "$ok" ]; then
+ # let's record the failure unless ignore is set
+ if [ -z "$IGNORE" ] ; then
+ FAILED_CMDS=("${FAILED_CMDS[@]}" "$TORECORD")
+ fi
fi
- REPO=$2
+ verbose "after run with $COMMAND" "ok=$ok" "#failed=${#FAILED_CMDS[@]}"
+ popd > /dev/null
+}
+
+function merge_all_branches () {
+ local REPO_DIR=$1; shift
+ local NAME=$1; shift
+ local REMOTE=$1; shift
- pushd ${REPO} > /dev/null
- eval $COMMAND
+ pushd $REPO_DIR > /dev/null
+ BRANCHES=$(git branch -r | grep $REMOTE | grep -v HEAD | sed "s/.*\///g" | grep -v master)
+ HAS_MASTER=$(git branch -l | grep master)
popd > /dev/null
+
+ [ -n "$HAS_MASTER" ] && run ${REPO_DIR} git checkout master
+ [ -n "$HAS_MASTER" ] && run ${REPO_DIR} git merge --ff $REMOTE/master
+ for BRANCH in $BRANCHES ; do
+ run -i ${REPO_DIR} git branch $BRANCH $REMOTE/$BRANCH
+ run ${REPO_DIR} git checkout $BRANCH
+ run ${REPO_DIR} git merge --ff $REMOTE/$BRANCH
+ done
}
-function mirror () {
- for arg in "$@" ; do
- NAME=$(basename ${arg} | sed s/.git$//g)
- GIT_NAME=${NAME}.git
- REPO_DIR=${LOCAL_MIRROR_DIR}/${NAME}
- MIRROR_REPO=${MIRROR_GIT}/${GIT_NAME}
- MASTER_REPO=${MASTER_GIT}/${GIT_NAME}
-
- # if there is no remote repository it may be that we only have
- # the repository locally and don't need to mirror
- git ls-remote $MIRROR_REPO &> /dev/null
- if [ $? -eq 0 ]
- then
- if [ -d ${REPO_DIR} ]
- then
- msg "pulling from ${NAME}"
- run "git fetch origin --tags" ${REPO_DIR}
- run "git fetch origin" ${REPO_DIR}
- run "git merge --ff origin/master" ${REPO_DIR}
- if [ $? -ne 0 ]
- then
- error "Can not fetch from ${MASTER_REPO}"
- fi
- else
- msg "mirroring ${NAME} for the first time"
- run "git clone ${MIRROR_REPO}" ${LOCAL_MIRROR_DIR}
- run "git remote add local_master ${MASTER_REPO}" ${REPO_DIR}
- fi
-
- msg "pushing ${NAME} to local master"
- run "git fetch local_master --tags" ${REPO_DIR}
- run "git fetch local_master" ${REPO_DIR}
- run "git merge --ff local_master/master" ${REPO_DIR}
- if [ $? -ne 0 ]
- then
- error "Can not fetch from ${MIRROR_REPO}"
- else
- run "git push local_master" ${REPO_DIR}
- run "git push --tags local_master" ${REPO_DIR}
- fi
- fi
+function push_all_branches () {
+ local REPO_DIR=$1; shift
+ local NAME=$1; shift
+ local PUSH_TO=$1; shift
+ local PUSH_FROM=$1; shift
+
+ pushd $REPO_DIR > /dev/null
+ BRANCHES=$(git branch -r | grep $PUSH_FROM | grep -v HEAD | sed "s/.*\///g" | grep -v master)
+ HAS_MASTER=$(git branch -l | grep master)
+ popd > /dev/null
+
+ [ -n "$HAS_MASTER" ] && run ${REPO_DIR} git push $PUSH_TO master:master
+ for BRANCH in $BRANCHES ; do
+ run ${REPO_DIR} git push $PUSH_TO $BRANCH:$BRANCH
done
}
+function mirror_repo () {
+ local repo=$1; shift
+ FAILED_CMDS=() # reset previous failure if any
+
+ NAME=$(basename ${repo} ".git")
+ GIT_NAME=${NAME}.git
+ REPO_DIR=${LOCAL_MIRROR_DIR}/${NAME}
+ REMOTE_REPO=${REMOTE_GIT}/${GIT_NAME}
+ MASTER_REPO=${MASTER_GIT}/${GIT_NAME}
+
+ # if the local master is a symlink (like /git/vserver-reference.git -> sliceref.git)
+ # then skip it
+ # we use this for either aliases (like vserver-reference and sliceimage) or
+ # for repos managed in other locations (like /git-slave) but where
+ # the symlink is needed so they get served by git-daemon
+ [ -h ${MASTER_REPO} ] && return
+
+ # if there is no remote repository it may be that we only have
+ # the repository locally and don't need to mirror
+ git ls-remote $REMOTE_REPO &> /dev/null || return
+
+ if [ -d ${REPO_DIR} ] ; then
+ msg "pulling from ${NAME}"
+ run ${REPO_DIR} git fetch origin --tags
+ run ${REPO_DIR} git fetch origin
+ merge_all_branches $REPO_DIR $NAME origin
+ if [ -n "$FAILED_CMDS" ]; then
+ # format mail body
+ body="Can not fetch from ${MASTER_REPO}\n\n------------\n FAILED COMMANDS:\n"
+ for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
+ notify $REPO_DIR "git-mirror.sh failed to merge remote with module ${NAME}" "$body"
+ return
+ fi
+ else
+ msg "mirroring ${NAME} for the first time"
+ run ${LOCAL_MIRROR_DIR} git clone ${REMOTE_REPO}
+ run ${REPO_DIR} git remote add local_master ${MASTER_REPO}
+ fi
-while getopts ":hq" opt
-do
+ msg "pushing ${NAME} to local master"
+ run ${REPO_DIR} git fetch local_master --tags
+ run ${REPO_DIR} git fetch local_master
+ merge_all_branches $REPO_DIR $NAME local_master
+ if [ -n "$FAILED_CMDS" ]; then
+ pushd ${REPO_DIR} > /dev/null
+ STATUS_OUT=$(git status)
+ popd > /dev/null
+ # format mail body
+ body="STATUS in ${REPO_DIR}:\n${STATUS_OUT} \n\n------------\n FAILED COMMANDS:\n"
+ for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
+ notify $REPO_DIR "git-mirror.sh failed on with module ${NAME}" "$body"
+ return
+ fi
+ run ${REPO_DIR} git push --tags local_master
+ push_all_branches $REPO_DIR $NAME local_master origin
+ if [ -n "$FAILED_CMDS" ]; then
+ # format mail body
+ body="FAILED COMMANDS:\n"
+ for line in "${FAILED_CMDS[@]}"; do body="$body$line\n"; done
+ notify $REPO_DIR "git-mirror.sh failed to push back with module ${NAME}" "$body"
+ return
+ fi
+ # success, remove previous check file if any
+ clear_notify $REPO_DIR
+ # touch a stamp so it's easier to figure out where/if things get stuck
+ touch ${REPO_DIR}/MIRRORED.stamp
+}
+
+function usage () {
+ echo "Usage $COMMAND [options] REPONAME*"
+ echo " [-a admin-mails] : provide space-separated admins emails"
+ echo " [-r remote-git-url] : e.g. -r git://git.onelab.eu/"
+ echo " [-q] quiet mode for running under cron"
+ echo " [-v] verbose mode"
+ echo " [-f] force mode, runs even if the lock file is present"
+ echo " see also manage-git-mirror.sh"
+ exit 1
+}
+
+while getopts "a:r:s:qvfh" opt; do
case $opt in
- q)
- QUIET=1
- break
- ;;
- h)
- echo "USAGE: $0 [-q] REPONAME*"
- exit 1
- ;;
- \?)
- echo "Invalid option: -$OPTARG" >&2
- ;;
+ a) ADMINS=$OPTARG ;;
+ r) REMOTE_GIT=$OPTARG ;;
+ q) QUIET=true ;;
+ v) VERBOSE=true ;;
+ f) FORCE=true ;;
+ h) usage ;;
+ \?) echo "Invalid option: -$opt" >&2 ;;
esac
done
-
shift $((OPTIND-1))
-mirror $@
+# skip this if force is set
+# the natural usage of force is, manage-git-mirror.sh touches the lock file
+# which prevents the cron jobs from running during an hour
+# then you can use with -f which leaves the lock file as it is
+if [ -z "$FORCE" ] ; then
+ # is the stamp older than an hour ?
+ GRACE=60
+ is_old=$(find $LOCKFILE -mmin +$GRACE 2> /dev/null)
+ if [ -n "$is_old" ] ; then
+ msg "$LOCKFILE is older than $GRACE minutes - removing"
+ rm -f $LOCKFILE
+ fi
+
+ if [ -f $LOCKFILE ] ; then
+ msg "Found $LOCKFILE - another git-mirror seems to be running. Aborting... "
+ exit 1
+ fi
+fi
+
+trap failure INT
+
+# if force is set we leave the lock file as is
+[ -z "$FORCE" ] && date > $LOCKFILE
+for gitrepo in "$@"; do mirror_repo $gitrepo ; done
+[ -z "$FORCE" ] && rm -f $LOCKFILE
git://git.onelab.eu / infrastructure.git / blobdiff