From c9b5cff75e6810b6de4cba9ed728b4b8789c91f6 Mon Sep 17 00:00:00 2001 From: "Kyle J. McKay" Date: Sat, 12 Sep 2020 22:37:38 -0700 Subject: [PATCH] gc-util-functions: refactor into two files Move utility functions used by gc that need to be used (in the near future) by update into a new updategc-util-functions.sh file and then source that from gc-util-functions.sh. This makes these functions available to update as well as gc without the overhead of pulling in all the unnecessary gc-specific stuff too. Signed-off-by: Kyle J. McKay --- jobd/gc-util-functions.sh | 126 +------------------------------------- jobd/updategc-util-functions.sh | 131 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 123 deletions(-) create mode 100755 jobd/updategc-util-functions.sh diff --git a/jobd/gc-util-functions.sh b/jobd/gc-util-functions.sh index 31ae8d7..3bbf24a 100755 --- a/jobd/gc-util-functions.sh +++ b/jobd/gc-util-functions.sh @@ -7,89 +7,13 @@ # this script without having already sourced shlib.sh [ -n "$var_git_exec_path" ] || exit 2 +# include functions used by both update and gc +. @basedir@/jobd/updategc-util-functions.sh + # default packing options packopts="--depth=50 --window=50 --window-memory=${var_window_memory:-1g}" quiet="-q"; [ "${show_progress:-0}" = "0" ] || quiet= -pidactive() { - if _result="$(kill -0 "$1" 2>&1)"; then - # process exists and we have permission to signal it - return 0 - fi - case "$_result" in *"not permitted"*) - # we do not have permission to signal the process - return 0 - esac - # process does not exist - return 1 -} - -createlock() { - # A .lock file should only exist for much less than a second. - # If we see a stale lock file (> 1h old), remove it and then, - # just in case, wait 30 seconds for any process whose .lock - # we might have just removed (it's racy) to finish doing what - # should take much less than a second to do. - _stalelock="$(find -L "$1.lock" -maxdepth 1 -mmin +60 -print 2>/dev/null)" || : - if [ -n "$_stalelock" ]; then - rm -f "$_stalelock" - sleep 30 - fi - for _try in p p n; do - if (set -C; >"$1.lock") 2>/dev/null; then - echo "$1.lock" - return 0 - fi - # delay and try again - [ "$_try" != "p" ] || sleep 1 - done - # cannot create lock file - return 1 -} - -# Create a gc.pid lockfile -# $1 => name of variable to store result in -# On success: -# variable named by $1 will contain the name of the newly create lockfile (i.e. "gc.pid") -# On failure: -# variable named by $1 will contain the failure reason -v_lock_gc() { - # be compatibile with gc.pid file from newer Git releases - _lockf='gc.pid' - _hn="$(hostname)" - _active= - if [ "$(createlock "$_lockf")" ]; then - # If $_lockf is: - # 1) less than 12 hours old - # 2) contains two fields (pid hostname) NO trailing NL - # 3) the hostname is different OR the pid is still alive - # then we exit as another active process is holding the lock - if [ "$(find -L "$_lockf" -maxdepth 1 -mmin -720 -print 2>/dev/null)" ]; then - _apid= - _ahost= - read -r _apid _ahost _ajunk <"$_lockf" || : - if [ "$_apid" ] && [ "$_ahost" ]; then - if [ "$_ahost" != "$_hn" ] || pidactive "$_apid"; then - _active=1 - fi - fi - fi - else - eval "$1="'"unable to create $_lockf.lock file"' - return 1 - fi - if [ -n "$_active" ]; then - rm -f "$_lockf.lock" - eval "$1="'"gc already running on machine '\''$_ahost'\'' pid '\''$_apid'\''"' - return 1 - fi - printf "%s %s" "$$" "$_hn" >"$_lockf.lock" - chmod 0664 "$_lockf.lock" - mv -f "$_lockf.lock" "$_lockf" - eval "$1="'"$_lockf"' - return 0 -} - # make sure combine-packs uses the correct Git executable run_combine_packs() { PATH="$var_git_exec_path:$cfg_basedir/bin:$PATH" @basedir@/jobd/combine-packs.sh "$@" @@ -107,50 +31,6 @@ combine_packs_std() { run_combine_packs --replace "$@" $packopts --all-progress-implied $quiet --non-empty } -# duplicate the first file to the name given by the second file making sure that -# the second file appears atomically all-at-once after the copy has been completed -# and does not appear at all if the copy fails (in which case this function fails) -# if the second file already exists this function fails with status 1 -# if the file names are the same this function returns immediately with success -dupe_file() { - [ "$1" != "$2" ] || return 0 - ! [ -e "$2" ] || return 1 - case "$2" in - *?/?*) _tmpdir="${2%/*}";; - *) _tmpdir=".";; - esac - _tmpfile="$(mktemp "${_tmpdir:-.}/packtmp-XXXXXX")" || return 1 - cp -fp "$1" "$_tmpfile" || return 1 - mv -f "$_tmpfile" "$2" -} - -# rename_pack oldnamepath newnamepath -# note that .keep and .bndl files are left untouched and not moved at all! -rename_pack() { - [ $# -eq 2 ] && [ "$1" != "$2" ] || { - echo >&2 "[$proj] incorrect use of rename_pack function" - exit 1 - } - # Git assumes that if the destination of the rename already exists - # that it is, in fact, a copy of the same bytes so silently succeeds - # without doing anything. We duplicate that logic here. - # Git checks for the .idx file first before even trying to use a pack - # so it should be the last moved and the first removed. - for ext in pack bitmap idx; do - [ -f "$1.$ext" ] || continue - ln "$1.$ext" "$2.$ext" >/dev/null 2>&1 || - dupe_file "$1.$ext" "$2.$ext" >/dev/null 2>&1 || - [ -f "$2.$ext" ] || { - echo >&2 "[$proj] unable to move $1.$ext to $2.$ext" - exit 1 - } - done - for ext in idx pack bitmap; do - rm -f "$1.$ext" - done - return 0 -} - # current directory must already be set to the $GIT_DIR # see if there are "lotsa" loose objects # "lotsa" is defined as the 17, 68, 71 and 86 object directories existing diff --git a/jobd/updategc-util-functions.sh b/jobd/updategc-util-functions.sh new file mode 100755 index 0000000..843a96b --- /dev/null +++ b/jobd/updategc-util-functions.sh @@ -0,0 +1,131 @@ +#!/bin/sh +# +# This is a shell library for common update/gc related functions +# used by various Girocco scripts. + +# shlib.sh always sets this, it's an error to source +# this script without having already sourced shlib.sh +[ -n "$var_git_exec_path" ] || exit 2 + +pidactive() { + if _result="$(kill -0 "$1" 2>&1)"; then + # process exists and we have permission to signal it + return 0 + fi + case "$_result" in *"not permitted"*) + # we do not have permission to signal the process + return 0 + esac + # process does not exist + return 1 +} + +createlock() { + # A .lock file should only exist for much less than a second. + # If we see a stale lock file (> 1h old), remove it and then, + # just in case, wait 30 seconds for any process whose .lock + # we might have just removed (it's racy) to finish doing what + # should take much less than a second to do. + _stalelock="$(find -L "$1.lock" -maxdepth 1 -mmin +60 -print 2>/dev/null)" || : + if [ -n "$_stalelock" ]; then + rm -f "$_stalelock" + sleep 30 + fi + for _try in p p n; do + if (set -C; >"$1.lock") 2>/dev/null; then + echo "$1.lock" + return 0 + fi + # delay and try again + [ "$_try" != "p" ] || sleep 1 + done + # cannot create lock file + return 1 +} + +# Create a gc.pid lockfile +# $1 => name of variable to store result in +# On success: +# variable named by $1 will contain the name of the newly create lockfile (i.e. "gc.pid") +# On failure: +# variable named by $1 will contain the failure reason +v_lock_gc() { + # be compatibile with gc.pid file from newer Git releases + _lockf='gc.pid' + _hn="$(hostname)" + _active= + if [ "$(createlock "$_lockf")" ]; then + # If $_lockf is: + # 1) less than 12 hours old + # 2) contains two fields (pid hostname) NO trailing NL + # 3) the hostname is different OR the pid is still alive + # then we exit as another active process is holding the lock + if [ "$(find -L "$_lockf" -maxdepth 1 -mmin -720 -print 2>/dev/null)" ]; then + _apid= + _ahost= + read -r _apid _ahost _ajunk <"$_lockf" || : + if [ "$_apid" ] && [ "$_ahost" ]; then + if [ "$_ahost" != "$_hn" ] || pidactive "$_apid"; then + _active=1 + fi + fi + fi + else + eval "$1="'"unable to create $_lockf.lock file"' + return 1 + fi + if [ -n "$_active" ]; then + rm -f "$_lockf.lock" + eval "$1="'"gc already running on machine '\''$_ahost'\'' pid '\''$_apid'\''"' + return 1 + fi + printf "%s %s" "$$" "$_hn" >"$_lockf.lock" + chmod 0664 "$_lockf.lock" + mv -f "$_lockf.lock" "$_lockf" + eval "$1="'"$_lockf"' + return 0 +} + +# duplicate the first file to the name given by the second file making sure that +# the second file appears atomically all-at-once after the copy has been completed +# and does not appear at all if the copy fails (in which case this function fails) +# if the second file already exists this function fails with status 1 +# if the file names are the same this function returns immediately with success +dupe_file() { + [ "$1" != "$2" ] || return 0 + ! [ -e "$2" ] || return 1 + case "$2" in + *?/?*) _tmpdir="${2%/*}";; + *) _tmpdir=".";; + esac + _tmpfile="$(mktemp "${_tmpdir:-.}/packtmp-XXXXXX")" || return 1 + cp -fp "$1" "$_tmpfile" || return 1 + mv -f "$_tmpfile" "$2" +} + +# rename_pack oldnamepath newnamepath +# note that .keep and .bndl files are left untouched and not moved at all! +rename_pack() { + [ $# -eq 2 ] && [ "$1" != "$2" ] || { + echo >&2 "[$proj] incorrect use of rename_pack function" + exit 1 + } + # Git assumes that if the destination of the rename already exists + # that it is, in fact, a copy of the same bytes so silently succeeds + # without doing anything. We duplicate that logic here. + # Git checks for the .idx file first before even trying to use a pack + # so it should be the last moved and the first removed. + for ext in pack bitmap idx; do + [ -f "$1.$ext" ] || continue + ln "$1.$ext" "$2.$ext" >/dev/null 2>&1 || + dupe_file "$1.$ext" "$2.$ext" >/dev/null 2>&1 || + [ -f "$2.$ext" ] || { + echo >&2 "[$proj] unable to move $1.$ext to $2.$ext" + exit 1 + } + done + for ext in idx pack bitmap; do + rm -f "$1.$ext" + done + return 0 +} -- 2.11.4.GIT