jobd/gc-util-functions.sh

   1 #!/bin/sh
   2 #
   3 # This is a shell library for common gc related functions
   4 # used by various Girocco scripts.
   5
   6 # shlib.sh always sets this, it's an error to source
   7 # this script without having already sourced shlib.sh
   8 [ -n "$var_git_exec_path" ] || exit 2
   9
  10 # include functions used by both update and gc
  11 . @basedir@/jobd/updategc-util-functions.sh
  12
  13 # default packing options
  14 packopts="--depth=50 --window=50 --window-memory=${var_window_memory:-1g}"
  15 quiet="-q"; [ "${show_progress:-0}" = "0" ] || quiet=
  16
  17 # make sure combine-packs uses the correct Git executable
  18 run_combine_packs() {
  19         PATH="$var_git_exec_path:$cfg_basedir/bin:$PATH" @basedir@/jobd/combine-packs.sh "$@"
  20 }
  21
  22 # combine the input pack(s) into a new pack (or possibly packs if packSizeLimit set)
  23 # input pack names are read from standard input one per line delimited by the first
  24 # ':', ' ' or '\n' character on the line (which allows gfi-packs to be read directly)
  25 # all arguments, if any, are passed to pack-objects as additional options
  26 # first removes any pre-existing "*.zap*" sentinels that may be leftover from any
  27 # previously aborted "--replace" operations
  28 # returns non-zero on failure
  29 combine_packs_std() {
  30         find -L objects/pack -maxdepth 1 -type f -name '*.zap*' -exec rm -f '{}' + || :
  31         run_combine_packs --replace "$@" $packopts --all-progress-implied $quiet --non-empty
  32 }
  33
  34 # current directory must already be set to the $GIT_DIR
  35 # see if there are "lotsa" loose objects
  36 # "lotsa" is defined as the 17, 68, 71 and 86 object directories existing
  37 # and there being at least 5 total objects between them which corresponds
  38 # to an approximate average of 320 loose objects before this function starts
  39 # returning true and triggering a "mini" gc to pack up loose objects
  40 lotsa_loose_objects() {
  41         [ -d objects/17 ] && [ -d objects/68 ] && [ -d objects/71 ] && [ -d objects/86 ] || return 1
  42         _objs=$(( $(find -L objects/17 objects/68 objects/71 objects/86 -maxdepth 1 -name "$octet19*" -type f -print 2>/dev/null | LC_ALL=C wc -l) ))
  43         [ ${_objs:-0} -ge 5 ]
  44 }
  45
  46 # same as lotsa_loose_objects but first runs `git prune-packed` if it can get a gc lock
  47 lotsa_loose_pruned_objects() {
  48         lotsa_loose_objects || return $?
  49         v_lock_gc _gclock || return 0
  50         git prune-packed --quiet
  51         rm -f "$_gclock"
  52         lotsa_loose_objects
  53 }
  54
  55 # a "single object" pack is either a pack containing just one object
  56 # or a pack containing zero objects (which is a degenerate case that
  57 # normally can never happen).  And it must have a name matching
  58 # the pack-<sha1>*.pack pattern where either the "infix" suffix is "_l"
  59 # or the name does not contain any "_" characters at all (and, obviously,
  60 # must have a matching .idx file).  Any .keep, .bundle, or .bitmap
  61 # associated packs are automatically excluded from the count.
  62 # "lotsa" here is defined as 20 or more.
  63 lotsa_single_object_packs() {
  64         __lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
  65         _lpo01="$__lpo --exclude-limit 2"
  66         # "$0=substr($0,19,length-23)" strips the leading "objects/pack/pack-" and trailing ".pack"
  67         _sopacks="$(
  68                 list_packs --quiet $_lpo01 objects/pack 2>/dev/null |
  69                 LC_ALL=C awk 'BEGIN {c=0} {$0=substr($0,19,length-23)} !/_/ || /^[0-9a-f]*_l$/ {c+=1} END {print c}'
  70         )" || :
  71         [ ${_sopacks:-0} -ge 20 ]
  72 }
  73
  74 # returns true if either lotsa_loose_pruned_objects or lotsa_single_object_packs is true
  75 lotsa_loose_objects_or_sopacks() {
  76         lotsa_single_object_packs || lotsa_loose_pruned_objects
  77 }
  78
  79 # pack any existing, non-packed loose objects into a new _l.pack file then run prune-packed
  80 # note that prune-packed is NOT run beforehand -- the caller must do that if needed
  81 # loose objects need not be part of complete commits/trees as --weak-naming is used
  82 # if there end up being too many loose packs, attempt to combine the packs too
  83 pack_incremental_loose_objects() {
  84         _lpacks="$(run_combine_packs </dev/null --names --loose --weak-naming --incremental --non-empty --all-progress-implied ${quiet:---progress} $packopts)"
  85         if [ -n "$_lpacks" ]; then
  86                 # We need to identify these packs later so we don't combine_packs them
  87                 for _objpack in $_lpacks; do
  88                         rename_pack "objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" || :
  89                 done
  90                 git prune-packed $quiet
  91         fi
  92         _packs=
  93         __lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
  94         _lpo01="$__lpo --exclude-limit 2"
  95         _lpol="$__lpo --exclude-no-sfx _l"
  96         list_packs --quiet $_lpo01 objects/pack 2>/dev/null |
  97         while read -r _apack && _apack="${_apack%.pack}" && [ -n "$_apack" ]; do
  98                 case "$_apack" in *_*);;*)
  99                         rename_pack "$_apack" "${_apack}_l" || :
 100                 esac
 101         done || :
 102         { _packs="$(list_packs --quiet --count $_lpol objects/pack || :)" || :; } 2>/dev/null
 103         [ "${_packs:-0}" -lt 20 ] || {
 104                 combine_small_incremental_loose_packs
 105                 _packs=
 106                 { _packs="$(list_packs --quiet --count $_lpol objects/pack || :)" || :; } 2>/dev/null
 107                 [ "${_packs:-0}" -lt 20 ] || combine_large_incremental_loose_packs
 108         }
 109 }
 110
 111 # same as pack_incremental_loose_objects except
 112 # returns true if locked and packed and unlocked or
 113 # false if could not lock (with err in $lockerr)
 114 pack_incremental_loose_objects_if_lockable() {
 115         if v_lock_gc _gclock; then
 116                 pack_incremental_loose_objects || :
 117                 rm -f "$_gclock"
 118                 return 0
 119         else
 120                 lockerr="$_gclock"
 121                 return 1
 122         fi
 123 }
 124
 125 # combine small _l packs into larger pack(s) using --weak-naming
 126 # we avoid any non _l, keep, bndl or bitmap packs
 127 combine_small_incremental_loose_packs() {
 128         _lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
 129         _lpo="$_lpo --exclude-no-sfx _l"
 130         _lpo="$_lpo --quiet --object-limit $var_redelta_threshold objects/pack"
 131         while
 132                 _cnt="$(list_packs --count $_lpo)" || :
 133                 test "${_cnt:-0}" -ge 2
 134         do
 135                 _newp="$(list_packs $_lpo | combine_packs_std --names --weak-naming --no-reuse-delta)"
 136                 # We need to identify these packs later so we don't combine_packs them
 137                 for _objpack in $_newp; do
 138                         rename_pack "objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" || :
 139                 done
 140                 v_cnt _newc $_newp
 141                 # be paranoid and exit the loop if we haven't reduced the number of packs
 142                 [ $_newc -lt $_cnt ] || break
 143         done
 144         return 0
 145 }
 146
 147 # combine large[ish] _l packs into larger pack(s) using --weak-naming
 148 # we avoid any non _l, keep, bndl or bitmap packs
 149 combine_large_incremental_loose_packs() {
 150         _lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
 151         _lpo="$_lpo --exclude-no-sfx _l"
 152         _lpo="$_lpo --quiet --exclude-limit -$(( ( $var_redelta_threshold / 2 ) + 1 )) objects/pack"
 153         while
 154                 _cnt="$(list_packs --count $_lpo)" || :
 155                 test "${_cnt:-0}" -ge 2
 156         do
 157                 _newp="$(list_packs $_lpo | combine_packs_std --names --weak-naming)"
 158                 # We need to identify these packs later so we don't combine_packs them
 159                 for _objpack in $_newp; do
 160                         rename_pack "objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" || :
 161                 done
 162                 v_cnt _newc $_newp
 163                 # be paranoid and exit the loop if we haven't reduced the number of packs
 164                 [ $_newc -lt $_cnt ] || break
 165         done
 166         return 0
 167 }