jobd/gc-util-functions.sh

   1 #!/bin/sh
   2 #
   3 # This is a shell library for common gc related functions
   4 # used by various Girocco scripts.
   5
   6 # shlib.sh always sets this, it's an error to source
   7 # this script without having already sourced shlib.sh
   8 [ -n "$var_git_exec_path" ] || exit 2
   9
  10 # include functions used by both update and gc
  11 . @basedir@/jobd/updategc-util-functions.sh
  12
  13 # default packing options
  14 packopts="--depth=50 --window=50 --window-memory=${var_window_memory:-1g}"
  15 quiet="-q"; [ "${show_progress:-0}" = "0" ] || quiet=
  16
  17 # Create a gc.pid lockfile
  18 # $1 => name of variable to store result in
  19 # On success:
  20 #   variable named by $1 will contain the name of the newly create lockfile (i.e. "gc.pid")
  21 # On failure:
  22 #   variable named by $1 will contain the failure reason
  23 v_lock_gc() {
  24         v_lock_file "$1" "gc.pid"
  25 }
  26
  27 # make sure combine-packs uses the correct Git executable
  28 run_combine_packs() {
  29         PATH="$var_git_exec_path:$cfg_basedir/bin:$PATH" @basedir@/jobd/combine-packs.sh "$@"
  30 }
  31
  32 # combine the input pack(s) into a new pack (or possibly packs if packSizeLimit set)
  33 # input pack names are read from standard input one per line delimited by the first
  34 # ':', ' ' or '\n' character on the line (which allows gfi-packs to be read directly)
  35 # all arguments, if any, are passed to pack-objects as additional options
  36 # first removes any pre-existing "*.zap*" sentinels that may be leftover from any
  37 # previously aborted "--replace" operations
  38 # returns non-zero on failure
  39 combine_packs_std() {
  40         find -L objects/pack -maxdepth 1 -type f -name '*.zap*' -exec rm -f '{}' + || :
  41         run_combine_packs --replace "$@" $packopts --all-progress-implied $quiet --non-empty
  42 }
  43
  44 # current directory must already be set to the $GIT_DIR
  45 # see if there are "lotsa" loose objects
  46 # "lotsa" is defined as the 17, 68, 71 and 86 object directories existing
  47 # and there being at least 5 total objects between them which corresponds
  48 # to an approximate average of 320 loose objects before this function starts
  49 # returning true and triggering a "mini" gc to pack up loose objects
  50 lotsa_loose_objects() {
  51         [ -d objects/17 ] && [ -d objects/68 ] && [ -d objects/71 ] && [ -d objects/86 ] || return 1
  52         _objs=$(( $(find -L objects/17 objects/68 objects/71 objects/86 -maxdepth 1 -name "$octet19*" -type f -print 2>/dev/null | LC_ALL=C wc -l) ))
  53         [ ${_objs:-0} -ge 5 ]
  54 }
  55
  56 # same as lotsa_loose_objects but first runs `git prune-packed` if it can get a gc lock
  57 lotsa_loose_pruned_objects() {
  58         lotsa_loose_objects || return $?
  59         v_lock_gc _gclock || return 0
  60         git prune-packed --quiet
  61         rm -f "$_gclock"
  62         lotsa_loose_objects
  63 }
  64
  65 # a "single object" pack is either a pack containing just one object
  66 # or a pack containing zero objects (which is a degenerate case that
  67 # normally can never happen).  And it must have a name matching
  68 # the pack-<sha1>*.pack pattern where either the "infix" suffix is "_l"
  69 # or the name does not contain any "_" characters at all (and, obviously,
  70 # must have a matching .idx file).  Any .keep, .bundle, or .bitmap
  71 # associated packs are automatically excluded from the count.
  72 # "lotsa" here is defined as 20 or more.
  73 lotsa_single_object_packs() {
  74         __lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
  75         _lpo01="$__lpo --exclude-limit 2"
  76         # "$0=substr($0,19,length-23)" strips the leading "objects/pack/pack-" and trailing ".pack"
  77         _sopacks="$(
  78                 list_packs --quiet $_lpo01 objects/pack 2>/dev/null |
  79                 LC_ALL=C awk 'BEGIN {c=0} {$0=substr($0,19,length-23)} !/_/ || /^[0-9a-f]*_l$/ {c+=1} END {print c}'
  80         )" || :
  81         [ ${_sopacks:-0} -ge 20 ]
  82 }
  83
  84 # returns true if either lotsa_loose_pruned_objects or lotsa_single_object_packs is true
  85 lotsa_loose_objects_or_sopacks() {
  86         lotsa_single_object_packs || lotsa_loose_pruned_objects
  87 }
  88
  89 # pack any existing, non-packed loose objects into a new _l.pack file then run prune-packed
  90 # note that prune-packed is NOT run beforehand -- the caller must do that if needed
  91 # loose objects need not be part of complete commits/trees as --weak-naming is used
  92 # if there end up being too many loose packs, attempt to combine the packs too
  93 pack_incremental_loose_objects() {
  94         _lpacks="$(run_combine_packs </dev/null --names --loose --weak-naming --incremental --non-empty --all-progress-implied ${quiet:---progress} $packopts)"
  95         if [ -n "$_lpacks" ]; then
  96                 # We need to identify these packs later so we don't combine_packs them
  97                 for _objpack in $_lpacks; do
  98                         rename_pack "objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" || :
  99                 done
 100                 git prune-packed $quiet
 101         fi
 102         _packs=
 103         __lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
 104         _lpo01="$__lpo --exclude-limit 2"
 105         _lpol="$__lpo --exclude-no-sfx _l"
 106         list_packs --quiet $_lpo01 objects/pack 2>/dev/null |
 107         while read -r _apack && _apack="${_apack%.pack}" && [ -n "$_apack" ]; do
 108                 case "$_apack" in *_*);;*)
 109                         rename_pack "$_apack" "${_apack}_l" || :
 110                 esac
 111         done || :
 112         { _packs="$(list_packs --quiet --count $_lpol objects/pack || :)" || :; } 2>/dev/null
 113         [ "${_packs:-0}" -lt 20 ] || {
 114                 combine_small_incremental_loose_packs
 115                 _packs=
 116                 { _packs="$(list_packs --quiet --count $_lpol objects/pack || :)" || :; } 2>/dev/null
 117                 [ "${_packs:-0}" -lt 20 ] || combine_large_incremental_loose_packs
 118         }
 119 }
 120
 121 # same as pack_incremental_loose_objects except
 122 # returns true if locked and packed and unlocked or
 123 # false if could not lock (with err in $lockerr)
 124 pack_incremental_loose_objects_if_lockable() {
 125         if v_lock_gc _gclock; then
 126                 pack_incremental_loose_objects || :
 127                 rm -f "$_gclock"
 128                 return 0
 129         else
 130                 lockerr="$_gclock"
 131                 return 1
 132         fi
 133 }
 134
 135 # combine small _l packs into larger pack(s) using --weak-naming
 136 # we avoid any non _l, keep, bndl or bitmap packs
 137 combine_small_incremental_loose_packs() {
 138         _lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
 139         _lpo="$_lpo --exclude-no-sfx _l"
 140         _lpo="$_lpo --quiet --object-limit $var_redelta_threshold objects/pack"
 141         while
 142                 _cnt="$(list_packs --count $_lpo)" || :
 143                 test "${_cnt:-0}" -ge 2
 144         do
 145                 _newp="$(list_packs $_lpo | combine_packs_std --names --weak-naming --no-reuse-delta)"
 146                 # We need to identify these packs later so we don't combine_packs them
 147                 for _objpack in $_newp; do
 148                         rename_pack "objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" || :
 149                 done
 150                 v_cnt _newc $_newp
 151                 # be paranoid and exit the loop if we haven't reduced the number of packs
 152                 [ $_newc -lt $_cnt ] || break
 153         done
 154         return 0
 155 }
 156
 157 # combine large[ish] _l packs into larger pack(s) using --weak-naming
 158 # we avoid any non _l, keep, bndl or bitmap packs
 159 combine_large_incremental_loose_packs() {
 160         _lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
 161         _lpo="$_lpo --exclude-no-sfx _l"
 162         _lpo="$_lpo --quiet --exclude-limit -$(( ( $var_redelta_threshold / 2 ) + 1 )) objects/pack"
 163         while
 164                 _cnt="$(list_packs --count $_lpo)" || :
 165                 test "${_cnt:-0}" -ge 2
 166         do
 167                 _newp="$(list_packs $_lpo | combine_packs_std --names --weak-naming)"
 168                 # We need to identify these packs later so we don't combine_packs them
 169                 for _objpack in $_newp; do
 170                         rename_pack "objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" || :
 171                 done
 172                 v_cnt _newc $_newp
 173                 # be paranoid and exit the loop if we haven't reduced the number of packs
 174                 [ $_newc -lt $_cnt ] || break
 175         done
 176         return 0
 177 }