admins.html: add lightweight irc web client link
[girocco.git] / jobd / gc-util-functions.sh
blobef47ef2989aa65b93ce88b32be6d2c8bacc48cdb
1 #!/bin/sh
3 # This is a shell library for common gc related functions
4 # used by various Girocco scripts.
6 # shlib.sh always sets this, it's an error to source
7 # this script without having already sourced shlib.sh
8 [ -n "$var_git_exec_path" ] || exit 2
10 # include functions used by both update and gc
11 . @basedir@/jobd/updategc-util-functions.sh
13 # default packing options
14 packopts="--depth=50 --window=50 --window-memory=${var_window_memory:-1g}"
15 quiet="-q"; [ "${show_progress:-0}" = "0" ] || quiet=
17 # Create a gc.pid lockfile
18 # $1 => name of variable to store result in
19 # On success:
20 # variable named by $1 will contain the name of the newly create lockfile (i.e. "gc.pid")
21 # On failure:
22 # variable named by $1 will contain the failure reason
23 v_lock_gc() {
24 v_lock_file "$1" "gc.pid"
27 # make sure combine-packs uses the correct Git executable
28 run_combine_packs() {
29 PATH="$var_git_exec_path:$cfg_basedir/bin:$PATH" @basedir@/jobd/combine-packs.sh "$@"
32 # combine the input pack(s) into a new pack (or possibly packs if packSizeLimit set)
33 # input pack names are read from standard input one per line delimited by the first
34 # ':', ' ' or '\n' character on the line (which allows gfi-packs to be read directly)
35 # all arguments, if any, are passed to pack-objects as additional options
36 # first removes any pre-existing "*.zap*" sentinels that may be leftover from any
37 # previously aborted "--replace" operations
38 # returns non-zero on failure
39 combine_packs_std() {
40 find -L objects/pack -maxdepth 1 -type f -name '*.zap*' -exec rm -f '{}' + || :
41 run_combine_packs --replace "$@" $packopts --all-progress-implied $quiet --non-empty
44 # current directory must already be set to the $GIT_DIR
45 # see if there are "lotsa" loose objects
46 # "lotsa" is defined as the 17, 68, 71 and 86 object directories existing
47 # and there being at least 5 total objects between them which corresponds
48 # to an approximate average of 320 loose objects before this function starts
49 # returning true and triggering a "mini" gc to pack up loose objects
50 lotsa_loose_objects() {
51 [ -d objects/17 ] && [ -d objects/68 ] && [ -d objects/71 ] && [ -d objects/86 ] || return 1
52 _objs=$(( $(find -L objects/17 objects/68 objects/71 objects/86 -maxdepth 1 -name "$octet19*" -type f -print 2>/dev/null | LC_ALL=C wc -l) ))
53 [ ${_objs:-0} -ge 5 ]
56 # same as lotsa_loose_objects but first runs `git prune-packed` if it can get a gc lock
57 lotsa_loose_pruned_objects() {
58 lotsa_loose_objects || return $?
59 v_lock_gc _gclock || return 0
60 git prune-packed --quiet
61 rm -f "$_gclock"
62 lotsa_loose_objects
65 # a "single object" pack is either a pack containing just one object
66 # or a pack containing zero objects (which is a degenerate case that
67 # normally can never happen). And it must have a name matching
68 # the pack-<sha1>*.pack pattern where either the "infix" suffix is "_l"
69 # or the name does not contain any "_" characters at all (and, obviously,
70 # must have a matching .idx file). Any .keep, .bundle, or .bitmap
71 # associated packs are automatically excluded from the count.
72 # "lotsa" here is defined as 20 or more.
73 lotsa_single_object_packs() {
74 __lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
75 _lpo01="$__lpo --exclude-limit 2"
76 # "$0=substr($0,19,length-23)" strips the leading "objects/pack/pack-" and trailing ".pack"
77 _sopacks="$(
78 list_packs --quiet $_lpo01 objects/pack 2>/dev/null |
79 LC_ALL=C awk 'BEGIN {c=0} {$0=substr($0,19,length-23)} !/_/ || /^[0-9a-f]*_l$/ {c+=1} END {print c}'
80 )" || :
81 [ ${_sopacks:-0} -ge 20 ]
84 # returns true if either lotsa_loose_pruned_objects or lotsa_single_object_packs is true
85 lotsa_loose_objects_or_sopacks() {
86 lotsa_single_object_packs || lotsa_loose_pruned_objects
89 # pack any existing, non-packed loose objects into a new _l.pack file then run prune-packed
90 # note that prune-packed is NOT run beforehand -- the caller must do that if needed
91 # loose objects need not be part of complete commits/trees as --weak-naming is used
92 # if there end up being too many loose packs, attempt to combine the packs too
93 pack_incremental_loose_objects() {
94 _lpacks="$(run_combine_packs </dev/null --names --loose --weak-naming --incremental --non-empty --all-progress-implied ${quiet:---progress} $packopts)"
95 if [ -n "$_lpacks" ]; then
96 # We need to identify these packs later so we don't combine_packs them
97 for _objpack in $_lpacks; do
98 rename_pack "objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" || :
99 done
100 git prune-packed $quiet
102 _packs=
103 __lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
104 _lpo01="$__lpo --exclude-limit 2"
105 _lpol="$__lpo --exclude-no-sfx _l"
106 list_packs --quiet $_lpo01 objects/pack 2>/dev/null |
107 while read -r _apack && _apack="${_apack%.pack}" && [ -n "$_apack" ]; do
108 case "$_apack" in *_*);;*)
109 rename_pack "$_apack" "${_apack}_l" || :
110 esac
111 done || :
112 { _packs="$(list_packs --quiet --count $_lpol objects/pack || :)" || :; } 2>/dev/null
113 [ "${_packs:-0}" -lt 20 ] || {
114 combine_small_incremental_loose_packs
115 _packs=
116 { _packs="$(list_packs --quiet --count $_lpol objects/pack || :)" || :; } 2>/dev/null
117 [ "${_packs:-0}" -lt 20 ] || combine_large_incremental_loose_packs
121 # same as pack_incremental_loose_objects except
122 # returns true if locked and packed and unlocked or
123 # false if could not lock (with err in $lockerr)
124 pack_incremental_loose_objects_if_lockable() {
125 if v_lock_gc _gclock; then
126 pack_incremental_loose_objects || :
127 rm -f "$_gclock"
128 return 0
129 else
130 lockerr="$_gclock"
131 return 1
135 # combine small _l packs into larger pack(s) using --weak-naming
136 # we avoid any non _l, keep, bndl or bitmap packs
137 combine_small_incremental_loose_packs() {
138 _lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
139 _lpo="$_lpo --exclude-no-sfx _l"
140 _lpo="$_lpo --quiet --object-limit $var_redelta_threshold objects/pack"
141 while
142 _cnt="$(list_packs --count $_lpo)" || :
143 test "${_cnt:-0}" -ge 2
145 _newp="$(list_packs $_lpo | combine_packs_std --names --weak-naming --no-reuse-delta)"
146 # We need to identify these packs later so we don't combine_packs them
147 for _objpack in $_newp; do
148 rename_pack "objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" || :
149 done
150 v_cnt _newc $_newp
151 # be paranoid and exit the loop if we haven't reduced the number of packs
152 [ $_newc -lt $_cnt ] || break
153 done
154 return 0
157 # combine large[ish] _l packs into larger pack(s) using --weak-naming
158 # we avoid any non _l, keep, bndl or bitmap packs
159 combine_large_incremental_loose_packs() {
160 _lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
161 _lpo="$_lpo --exclude-no-sfx _l"
162 _lpo="$_lpo --quiet --exclude-limit -$(( ( $var_redelta_threshold / 2 ) + 1 )) objects/pack"
163 while
164 _cnt="$(list_packs --count $_lpo)" || :
165 test "${_cnt:-0}" -ge 2
167 _newp="$(list_packs $_lpo | combine_packs_std --names --weak-naming)"
168 # We need to identify these packs later so we don't combine_packs them
169 for _objpack in $_newp; do
170 rename_pack "objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" || :
171 done
172 v_cnt _newc $_newp
173 # be paranoid and exit the loop if we haven't reduced the number of packs
174 [ $_newc -lt $_cnt ] || break
175 done
176 return 0