From ad6956af739853d8d63023549e64bd378f18cd2d Mon Sep 17 00:00:00 2001 From: "Kyle J. McKay" Date: Tue, 9 Jun 2020 19:50:11 -0700 Subject: [PATCH] gc-git-svn: arrange for `git-svn fetch` to run gc-git-svn.sh When `git svn fetch` runs `git gc --auto`, arrange for it to end up running the `jobd/gc-git-svn.sh` script. This will allow Girocco to perform any needed loose object packing during a long `git-svn fetch` operation. Up until now, all garbage collection has been entirely suppressed during `git-svn fetch` operations potentially resulting in an excessively large number of loose objects being created before they were ultimately packed up. The `gc-git-svn.sh` script currently does nothing but will be filled in by a future commit. Signed-off-by: Kyle J. McKay --- hooks/pre-auto-gc | 6 ++++++ jobd/gc-git-svn.sh | 26 ++++++++++++++++++++++++++ jobd/update.sh | 13 +++++++++++++ shlib.sh | 1 + taskd/clone.sh | 13 +++++++++++++ 5 files changed, 59 insertions(+) create mode 100755 jobd/gc-git-svn.sh diff --git a/hooks/pre-auto-gc b/hooks/pre-auto-gc index e224a6f..98769bf 100755 --- a/hooks/pre-auto-gc +++ b/hooks/pre-auto-gc @@ -8,6 +8,12 @@ # If GIROCCO_SUPPRESS_AUTO_GC_UPDATE immediately exit suppressing auto gc [ "${GIROCCO_SUPPRESS_AUTO_GC_UPDATE:-0}" = "0" ] || exit 1 +# If GIROCCO_DIVERT_GIT_SVN_AUTO_GC divert to $basedir/jobd/gc-git-svn.sh +[ "${GIROCCO_DIVERT_GIT_SVN_AUTO_GC:-0}" = "0" ] || { + @basedir@/jobd/gc-git-svn.sh + exit 1 +} + # If we're in the chroot immediately exit suppressing auto gc [ -x @perlbin@ ] || exit 1 diff --git a/jobd/gc-git-svn.sh b/jobd/gc-git-svn.sh new file mode 100755 index 0000000..80c5e0d --- /dev/null +++ b/jobd/gc-git-svn.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# Girocco installs a pre-auto-gc hook. +# When $GIROCCO_DIVERT_GIT_SVN_AUTO_GC is set to 1 +# that hook, when executed, runs this script and +# no matter what exit code this script uses, the +# pre-auto-gc hook always exits with a code of 1 +# immediately after this script finishes thereby +# aborting any Git-directed gc. + +# The `git-svn fetch` operation periodically runs +# `git gc --auto` which ultimately causes this +# script to run provided the correct conditions +# are met and $GIROCCO_DIVERT_GIT_SVN_AUTO_GC is +# set to 1 in the environment. + +# Both the jobd/update.sh and taskd/clone.sh scripts +# set and export GIROCCO_DIVERT_GIT_SVN_AUTO_GC=1 +# and force `gc.auto=1` as well whenever they process +# a `git-svn` mirror. + +. @basedir@/shlib.sh + +set -e + +exit 0 diff --git a/jobd/update.sh b/jobd/update.sh index f6c86a5..794da5c 100755 --- a/jobd/update.sh +++ b/jobd/update.sh @@ -313,7 +313,20 @@ case "$url" in # remove any stale *.lock files greater than 1 hour old in case # git-svn was killed on the last update because it took too long find -L svn -type f -name '*.lock' -mmin +60 -exec rm -f '{}' + 2>/dev/null || : + GIROCCO_DIVERT_GIT_SVN_AUTO_GC=1 + export GIROCCO_DIVERT_GIT_SVN_AUTO_GC + unset GIROCCO_SUPPRESS_AUTO_GC_UPDATE + saveconfig="$GIT_CONFIG_PARAMETERS" + git_add_config 'gc.auto=1' GIT_DIR=. bang git svn fetch --log-window-size=$var_log_window_size --username="$svnuser" --quiet <"$mtlinesfile" + GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1 + export GIROCCO_SUPPRESS_AUTO_GC_UPDATE + unset GIROCCO_DIVERT_GIT_SVN_AUTO_GC + unset GIT_CONFIG_PARAMETERS + [ -z "$saveconfig" ] || { + GIT_CONFIG_PARAMETERS="$saveconfig" + export GIT_CONFIG_PARAMETERS + } # git svn does not preserve group permissions in the svn subdirectory chmod -R ug+rw,o+r svn # git svn also leaves behind ref turds that end with @nnn diff --git a/shlib.sh b/shlib.sh index b65bb1e..6dcd432 100644 --- a/shlib.sh +++ b/shlib.sh @@ -240,6 +240,7 @@ if [ -n "$defined_cfg_git_client_ua" ]; then export GIT_USER_AGENT fi unset GIT_CONFIG_PARAMETERS +unset GIROCCO_DIVERT_GIT_SVN_AUTO_GC # ## IMPORTANT! diff --git a/taskd/clone.sh b/taskd/clone.sh index c3d10aa..c7db20b 100755 --- a/taskd/clone.sh +++ b/taskd/clone.sh @@ -369,8 +369,21 @@ case "$url" in while [ "$svn_retries" -gt 0 ]; do svn_retries="$(( $svn_retries - 1 ))" svn_err=0 + GIROCCO_DIVERT_GIT_SVN_AUTO_GC=1 + export GIROCCO_DIVERT_GIT_SVN_AUTO_GC + unset GIROCCO_SUPPRESS_AUTO_GC_UPDATE + saveconfig="$GIT_CONFIG_PARAMETERS" + git_add_config 'gc.auto=1' # Again, be careful to use GIT_DIR=. here or else new .git subdirectory! GIT_DIR=. git svn fetch --log-window-size=$var_log_window_size --username="$svnuser" --quiet <"$mtlinesfile" || svn_err="$?" + GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1 + export GIROCCO_SUPPRESS_AUTO_GC_UPDATE + unset GIROCCO_DIVERT_GIT_SVN_AUTO_GC + unset GIT_CONFIG_PARAMETERS + [ -z "$saveconfig" ] || { + GIT_CONFIG_PARAMETERS="$saveconfig" + export GIT_CONFIG_PARAMETERS + } [ "${svn_err:-1}" -ne 0 ] || break # success! # Check to see if we made any progress v_get_svn_progress_fingerprint svn_progress_now -- 2.11.4.GIT