jobd/gc.sh: switch the gc "signature" hash to use ltsha256
[girocco.git] / jobd / gc.sh
blob877f43e9b860d6e007eb8a86c786d880218d8d0f
1 #!/bin/sh
3 # NOTE: additional options can be passed to git repack by specifying
4 # them after the project name, for example:
5 # gc.sh my-project -f
7 . @basedir@/shlib.sh
8 . @basedir@/jobd/gc-util-functions.sh
10 set -e
12 if [ $# -lt 1 ]; then
13 echo "Usage: gc.sh projname [extra-repack-args]" >&2
14 exit 1
17 # Includes
18 _shlib_done=1
19 unset GIROCCO_SUPPRESS_AUTO_GC_UPDATE
20 . "$cfg_basedir/jobd/maintain-auto-gc-hack.sh"
21 . "$cfg_basedir/jobd/generate-auto-gc-update.sh"
22 GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1 && export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
24 umask 002
25 [ "$cfg_permission_control" != "Hooks" ] || umask 000
26 clean_git_env
28 # The pre-receive script creates one ref log file per push but we want them to
29 # be coalesced into one ref log file per day. We are guaranteed that any files
30 # we find to coalesce are NOT currently being written to since they are always
31 # written first as temporary files and then moved into place. We attempt to
32 # transfer the most recent modification time to the coalesced log file which
33 # would step on its mod time if it were being written to directly, but if we
34 # find per-process ref log files then it must be a push project and the only
35 # thing that would write directly to the main per-day log file would be a
36 # mirror project so there's actually no conflict.
37 # Also, if the clock is wonky (or was futzed with) we may have both YYYYMMDD
38 # and YYYYMMDD.gz present in which case combine them into YYYYMMDD
39 coalesce_reflogs() {
40 [ -d reflogs ] || return 0
41 rm -f .gc_failed
42 find -L reflogs -maxdepth 1 -type f -name "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]" -print |
43 while read -r rname; do
44 if [ -e "$rname.gz" ]; then
45 if [ -s "$rname" ]; then
46 # Presumably the .gz file must have been created before the non-gz
47 # file since it had to be uncompressed at some point therefore
48 # we need to append the non-gz contents to it but keep the non-gz
49 # contents timestamp so we rename to YYYYMMDD_ which will sort first
50 # and be picked up in the next step if we are interrupted in the middle.
51 # If a YYYYMMDD_ file already exists we append to it and transfer the
52 # timestamp. Finally we transfer the YYYYMMDD_ timestamp to the result
53 # and remove the YYYYMMDD_ temporary file leaving the result uncompressed.
54 if [ -e "${rname}_" ]; then
55 cat "$rname" >>"${rname}_"
56 touch -r "$rname" "${rname}_"
57 rm -f "$rname"
58 ! [ -e "$rname" ]
59 else
60 mv "$rname" "${rname}_"
62 gzip -d "$rname.gz" </dev/null
63 [ -e "$rname" ] && ! [ -e "$rname.gz" ]
64 cat "${rname}_" >>"$rname"
65 touch -r "${rname}_" "$rname"
66 rm -f "${rname}_"
67 else
68 # Just remove the empty file to resolve the problem
69 rm -f "$rname"
72 done
73 find -L reflogs -maxdepth 1 -type f -name "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_*" -print | LC_ALL=C sort |
74 while read -r rname; do
75 logname="${rname%%_*}"
76 # If someone's been futzing with the date, the file we want to
77 # append to could already have been compressed, so we just uncompress
78 # it here. The previous block guarantees we do not have both a compressed
79 # and uncompressed version present at the same time.
80 if [ -e "$logname.gz" ]; then
81 gzip -d "$logname.gz" </dev/null
82 [ -e "$logname" ] && ! [ -e "$logname.gz" ]
84 cat "$rname" >>"$logname"
85 touch -r "$rname" "$logname"
86 rm -f "$rname"
87 if [ -e "$rname" ]; then
88 >.gc_failed
89 echo "! [$proj] failed to remove $rname" >&2
90 exit 1 # will only exit subshell created by "|"
92 done
93 ! [ -e .gc_failed ]
96 # Remove any files in reflogs that are older than $cfg_reflogs_lifetime days
97 prune_reflogs() {
98 [ -d reflogs ] || return 0
99 exp="$(( ${cfg_reflogs_lifetime:-1} * 1440 ))"
100 [ $exp -gt 0 ] || exp=1440
101 [ $exp -le 43200 ] || exp=43200
102 find -L reflogs -maxdepth 1 -type f -mmin "+$exp" -exec rm -f '{}' + || :
105 # Compact any reflogs that are not today's UTC date unless a .gz version exists
106 compact_reflogs() {
107 [ -d reflogs ] || return 0
108 _td="reflogs/$(TZ=UTC date '+%Y%m%d')"
109 find -L reflogs -maxdepth 1 -type f -name "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]" -print |
110 while read -r rname; do
111 [ "$rname" != "$_td" ] || continue
112 ! [ -e "$rname.gz" ] || continue
113 gzip -9 "$rname" </dev/null
114 done
117 # return true if there's more than one objects/pack-<sha>.pack file or
118 # ANY sha-1 files in objects or
119 # there's one pack and it's not a normal pack name or
120 # there's one pack but not any refs
121 is_dirty() {
122 _packs="$(find -L objects/pack -name "pre-auto-gc-[12].pack" -prune -o -name "*.pack" -type f -print 2>/dev/null | head -n 2)"
123 v_cnt _packscnt $_packs
124 if [ $_packscnt -gt 1 ]; then
125 return 0
127 if [ $_packscnt -eq 1 ]; then
128 # the single pack name is in $_packs
129 _packs="${_packs%.pack}"
130 _packs="${_packs#objects/pack/}"
131 case "$_packs" in
132 pack-*)
133 _packs="${_packs#pack-}"
134 if [ "${#_packs}" -lt 40 ] || [ "${_packs#*[!0-9a-fA-F]}" != "$_packs" ]; then
135 # name not exclusively 40 or more hexadecimal digits makes it dirty
136 return 0
140 # abnormal name makes it dirty
141 return 0
143 esac
145 _objs=$(find -L objects/$octet -name "$octet19*" -type f -print 2>/dev/null | head -n 1 | LC_ALL=C wc -l)
146 [ $_objs -eq 0 ] || return 0
147 [ $_packscnt -eq 1 ] || return 1
148 # we do this check last because it's potentially the most expensive;
149 # at this point we know we do not have any loose objects, but we do
150 # have one pack that's named "normally"; empty refs => dirty
151 is_empty_refs_dir
154 # combine the input pack(s) into a new pack (or possibly packs if packSizeLimit set)
155 # input pack names are read from standard input one per line delimited by the first
156 # ':', ' ' or '\n' character on the line (which allows gfi-packs to be read directly)
157 # all arguments, if any, are passed to pack-objects as additional options
158 # returns non-zero on failure AND creates .gc_failed in that case
159 combine_packs() {
160 rm -f .gc_failed
161 combine_packs_std "$@" || {
162 >.gc_failed
163 return 1
165 return 0
168 # if the current directory is_gfi_mirror then repack all packs listed in gfi-packs
169 repack_gfi_packs() {
170 [ -n "$gfi_mirror" ] || return 0
171 [ -d objects/pack ] || { rm -f gfi-packs; return 0; }
172 progress "~ [$proj] redeltifying poor quality git fast-import packs"
173 combine_packs --ignore-missing --no-reuse-delta <gfi-packs
174 rm -f gfi-packs
175 return 0
178 # pack any existing loose objects into a new _l.pack file then run prune-packed
179 # note that prune-packed is NOT run beforehand -- the caller must do that if needed
180 # loose objects need not be part of complete commits/trees as --weak-naming is used
181 pack_loose_objects() {
182 _lpacks="$(run_combine_packs </dev/null --names --loose --weak-naming --non-empty --all-progress-implied ${quiet:---progress} $packopts)"
183 if [ -n "$_lpacks" ]; then
184 # We need to identify these packs later so we don't combine_packs them
185 for _objpack in $_lpacks; do
186 rename_pack "objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" || :
187 done
188 git prune-packed $quiet
192 # combine small packs into larger pack(s)
193 # we avoid any _[lo], keep, bndl or bitmap packs
194 # if the optional argument is non-empty even a single small pack will be redeltad
195 combine_small_packs() {
196 _didprogress=
197 _minsmallpacks=2
198 if [ -n "$1" ] && [ -n "$noreusedeltaopt" ]; then
199 _minsmallpacks=1
201 _lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
202 _lpo="$_lpo --exclude-sfx _u --exclude-sfx _o --exclude-sfx _l"
203 _lpo="$_lpo --quiet --object-limit $var_redelta_threshold objects/pack"
204 while
205 _cnt="$(list_packs --count $_lpo)" || :
206 test "${_cnt:-0}" -ge $_minsmallpacks
208 [ -n "$_didprogress" ] || {
209 progress "~ [$proj] combining small packs into a single larger pack"
210 _didprogress=1
212 _newp="$(list_packs $_lpo | combine_packs --names $noreusedeltaopt)"
213 v_cnt _newc $_newp
214 # be paranoid and exit the loop if we haven't reduced the number of packs
215 [ $_newc -lt $_cnt ] || break
216 _minsmallpacks=2
217 done
218 return 0
221 # combine small _l packs into larger pack(s) using --weak-naming
222 # we avoid any non _l, keep, bndl or bitmap packs
223 # if the optional 2nd argument is non-empty even a single small pack will be redeltad
224 combine_small_loose_packs() {
225 _didprogress=
226 _minsmallpacks=2
227 if [ -n "$1" ] && [ -n "$noreusedeltaopt" ]; then
228 _minsmallpacks=1
230 _lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
231 _lpo="$_lpo --exclude-no-sfx _l"
232 _lpo="$_lpo --quiet --object-limit $var_redelta_threshold objects/pack"
233 while
234 _cnt="$(list_packs --count $_lpo)" || :
235 test "${_cnt:-0}" -ge $_minsmallpacks
237 [ -n "$_didprogress" ] || {
238 progress "~ [$proj] combining small loose packs into a single larger pack"
239 _didprogress=1
241 _newp="$(list_packs $_lpo | combine_packs --names --weak-naming $noreusedeltaopt)"
242 # We need to identify these packs later so we don't combine_packs them
243 for _objpack in $_newp; do
244 rename_pack "objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" || :
245 done
246 v_cnt _newc $_newp
247 # be paranoid and exit the loop if we haven't reduced the number of packs
248 [ $_newc -lt $_cnt ] || break
249 _minsmallpacks=2
250 done
251 return 0
254 # Unfortunately, some fetch strategies (e.g. git-svn and non-smart HTTP) lack
255 # the ability to store newly fetched objects in a pack.
256 # However, the fetch code conveniently sets .needspack just before it fetches
257 # so that it's easy to find all the loose objects that have been fetched and
258 # combine them into a pack. The --no-reuse-delta option is meaningless here
259 # since everything to be packed is a loose object and therefore not a delta so
260 # deltification will always take place.
261 make_needs_pack() {
262 [ -f .needspack ] || return 0
263 rm -f .needspackgc
264 mv -f .needspack .needspackgc
265 progress "~ [$proj] combining fetched loose objects into a pack"
266 _newp="$(find -L objects/$octet -maxdepth 1 -type f -newer .needspackgc -name "$octet19*" -print 2>/dev/null |
267 LC_ALL=C awk -F / '{print $2 $3}' |
268 run_combine_packs --objects --names $packopts --incremental --all-progress-implied $quiet --non-empty)" || {
269 # We used to fail gc here.
270 # Now, however, we just ignore the failure because we have
271 # another mechanism to handle loose objects and it's possible
272 # that the fetcher somehow brought in unconnected objects which
273 # would cause the above combine-packs to fail.
274 # By ignoring the failure and just removing the .needspack file
275 # the loose objects will be treated as "ordinary" loose objects
276 # and packed using the "--weak-naming" option which can handle
277 # broken connectivity.
278 # That's a better solution than just failing here or leaving
279 # .needspack behind to potentially continue to fail again and
280 # again.
281 _newp=
283 if [ -n "$_newp" ]; then
284 # remove the now-redundant loose objects -- this is always safe
285 # even during a concurrent push because a reprepare_packed_git
286 # will be triggered if an object that should be there is not
287 # found thereby finding it in the new pack instead
288 git prune-packed $quiet
290 rm -f .needspackgc
293 # HEADSHA="$(pack_is_complete /full/path/to/some.pack /full/path/to/packed-refs "$(cat HEAD)")"
294 pack_is_complete() {
295 # Must have a matching .idx file and a non-empty packed-refs file
296 [ -s "${1%.pack}.idx" ] || return 1
297 [ -s "$2" ] || return 1
298 _headsha=
299 case "$3" in
300 $octet20*)
301 _headsha="$3"
303 "ref: refs/"?*|"ref:refs/"?*|"refs/"?*)
304 _headmatch="${3#ref:}"
305 _headmatch="${_headmatch# }"
306 _headmatchpat="$(echo "$_headmatch" | LC_ALL=C sed -e 's/\([.$]\)/\\\1/g')"
307 _headsha="$(LC_ALL=C grep -e "^$octet20$hexdig* $_headmatchpat\$" <"$2" |
308 LC_ALL=C cut -d ' ' -f 1)"
309 case "$_headsha" in $octet20*) :;; *)
310 return 1
311 esac
314 # bad HEAD
315 return 1
316 esac
317 rm -rf pack_is_complete_test
318 mkdir pack_is_complete_test
319 mkdir pack_is_complete_test/refs
320 mkdir pack_is_complete_test/objects
321 mkdir pack_is_complete_test/objects/pack
322 echo "$_headsha" >pack_is_complete_test/HEAD
323 ln -s "$1" pack_is_complete_test/objects/pack/
324 ln -s "${1%.pack}.idx" pack_is_complete_test/objects/pack/
325 ln -s "$2" pack_is_complete_test/packed-refs
326 _count="$(git --git-dir=pack_is_complete_test rev-list --count --all 2>/dev/null)" || :
327 rm -rf pack_is_complete_test
328 [ -n "$_count" ] || return 1
329 [ "$_count" -gt 0 ] 2>/dev/null || return 1
330 echo "$_headsha"
333 # write the hexadecimal sha256 hash of stdin plus a trailing newline to stdout
334 sig_hash_stdin() {
335 "$cfg_basedir/bin/ltsha256"
338 # Compute a "signature" for the refs in the specified (default .) repository.
339 # Instead of passing --head to show-ref, the actual contents of HEAD are used
340 # to capture changes in HEAD's symbolic-ref value that would otherwise be missed.
341 # $1 => variable name to store result hash in
342 # $2 => --git-dir to use, defaults to "."
343 v_compute_refs_sighash() {
344 eval "$1="'"$({ cat "${2:-.}/HEAD"; git --git-dir="${2:-.}" show-ref; } |
345 sig_hash_stdin)"'
348 # On return a "$lockf" will have been created that must be removed when gc is done
349 lock_gc() {
350 v_lock_gc _lockresult || {
351 echo >&2 "[$proj] $_lockresult"
352 exit 1
354 lockf="$_lockresult"
357 # Create a repack subdirectory such that running repack in it will pack the
358 # same things that a pack in the normal directory would except that the pack
359 # is guaranteed to be generated in an optimized order by adding a suitable
360 # synthesized ref in the refs/tags namespace (yes, pack-objects.c really does
361 # behave differently depending on the contents of the refs/tags namespace).
362 # Before calling this, pack-refs --all MUST be performed or the wrong pack
363 # will end up being made.
365 # If a ref deletion is pushed after making the repack subdir but before the
366 # the actual repack, the discarded objects will be packed -- no big deal,
367 # they'll get discarded the next time gc runs.
369 # If a fast-forward ref update is pushed after making the repack subdir but
370 # before the actual repack, it will be picked up and the new objects packed
371 # (subject to the normal git repack race about picking such updates up).
373 # If a non-fast-forward ref update is pushed after making the repack subdir but
374 # before the actual repack, it will be picked up like a fast-forward update but
375 # the discarded objects will be included like a ref deletion (until the next
376 # scheduled gc takes place).
378 # We retain a copy of the original packed-refs file as repack/packed-refs.orig
379 # If ref deletions come in while we're repacking, the original packed-refs
380 # file will be modified, but we'll still pack the deleted ref(s).
381 # If the packed-refs.orig file is used to create the bundle header we avoid
382 # a situation where the bundle contains a ref state that never actually
383 # existed in reality (for example a new branch is pushed and then an old
384 # branch deleted afterwards -- the deletion would show up in the bundle
385 # because it will cause the original packed-refs file to be re-written, but
386 # the new branch creation will not unless we do another pack-refs which might
387 # lead to having in incomplete bundle). Therefore we want to keep a copy of
388 # the original packed-refs file around. We do the same thing for HEAD.
390 # It's possible that the "objects" subdirectory is a symbolic link.
391 # Git does support this. However, during the repacking process, new packs
392 # will be created in repack/alt/pack and then moved into objects/pack.
393 # In order for this to work seemlessly, they must both be on the same
394 # filesystem. But when objects (or even objects/pack) is a symbolic link they
395 # might not be. For this reason a "repack" subdirectory is created under
396 # objects/pack and the repack/alt/pack directory symbolicly linked to it.
398 # Git allows not just HEAD to be a symbolic-ref, but any ref anywhere in the
399 # refs namespace. We are concerned about ref name collisions and getting the
400 # right tag set to get an optimal pack. We can safely duplicate the ref space
401 # under refs/heads, refs/notes and refs/remotes without any risk of unwanted
402 # collisions and this will likely make over 99%+ of all symbolic refs found
403 # in the wild work properly. Girocco itself never creates any symbolic refs
404 # inside the refs namespace; this is a nod to simultaneously using a Girocco
405 # repository for other purposes.
406 make_repack_dir() {
407 ! [ -d repack ] || rm -rf repack
408 ! [ -d repack ] || { echo >&2 "[$proj] cannot remove repack subdirectory"; exit 1; }
409 [ -d objects/pack ] || mkdir -p objects/pack
410 ! [ -d objects/pack/repack ] || rm -rf objects/pack/repack
411 ! [ -d objects/pack/repack ] || { echo >&2 "[$proj] cannot remove objects/pack/repack subdirectory"; exit 1; }
412 mkdir repack repack/refs repack/alt objects/pack/repack
413 [ -d info ] || mkdir info
414 ln -s ../config repack/config
415 ln -s ../info repack/info
416 ln -s ../objects repack/objects
417 ln -s "$PWD/objects/pack/repack" repack/alt/pack
418 ! [ -d logs ] || ln -s ../logs repack/logs
419 ! [ -d worktrees ] || ln -s ../worktrees repack/worktrees
420 _lines=$(( $(LC_ALL=C wc -l <packed-refs) ))
421 cat HEAD >repack/HEAD.orig
422 cat packed-refs >repack/packed-refs.orig
423 ln -s packed-refs.orig repack/packed-refs
424 cp repack/HEAD.orig repack/HEAD
425 v_compute_refs_sighash _rsig repack
426 rm repack/packed-refs repack/HEAD
427 ln -s ../../refs repack/refs/refs
428 echo "$_rsig" >repack/packed-refs.sighash
429 >repack/packed-refs.extra
430 _xtralines=0
431 if [ $(LC_ALL=C wc -l <repack/packed-refs.orig) -ne "$_lines" ]; then
432 echo >&2 "[$proj] error: make_repack_dir failed original packed-refs line count sanity check"
433 exit 1
435 if [ "${cfg_fetch_stash_refs:-0}" = "0" ]; then
436 # migrate any refs/stash or refs/tgstash lines to repack/packed-refs.extra
437 <repack/packed-refs.orig LC_ALL=C awk -v xtra="repack/packed-refs.extra" '
438 BEGIN { peeling = 0 }
439 NR == 1 && /^#/ { print; next; }
440 peeling && /^\^/ { print >>xtra; next; }
441 /^[0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f]+ refs\/(stash|tgstash)(\/|$)/ {
442 peeling = 1
443 print >>xtra
444 next
446 { peeling = 0; print; }
447 ' >repack/packed-refs.new
448 _xtralines="$(( $(LC_ALL=C wc -l <repack/packed-refs.extra) + 0 ))"
449 _newlines="$(( $(LC_ALL=C wc -l <repack/packed-refs.new) + 0 ))"
450 if [ "$(( $_newlines + $_xtralines ))" -ne "$_lines" ]; then
451 echo >&2 "[$proj] error: make_repack_dir failed packed-refs.extra line count sanity check"
452 exit 1
454 mv -f repack/packed-refs.new repack/packed-refs.orig
455 _lines="$_newlines"
457 # Note: Git v1.5.0 introduced the "# pack-refs with:" header line for the packed-refs file
458 sed '/^# pack-refs/d; s, refs/, refs/!/,' <repack/packed-refs.orig >repack/packed-refs
459 nohead=
460 headref="$(git rev-parse --verify --quiet HEAD)" || :
461 if [ -n "$headref" ]; then
462 echo "$headref refs/!=/HEAD" >>repack/packed-refs
463 echo "$headref refs/heads/!" >>repack/packed-refs
464 nohead='\, refs/heads/!$,d; '
465 _lines=$(( $_lines + 2 ))
467 if [ $(( $(LC_ALL=C wc -l <repack/packed-refs) + 1 )) -ne "$_lines" ]; then
468 echo >&2 "[$proj] error: make_repack_dir failed packed-refs initial line count sanity check"
469 exit 1
471 sed -n "$nohead"'\, refs/heads/,p; \, refs/notes/,p; \, refs/remotes/,p' <repack/packed-refs.orig >>repack/packed-refs
472 _newlines="$(( $(LC_ALL=C wc -l <repack/packed-refs) ))"
473 if [ $(( $_newlines + 1 )) -lt "$_lines" ]; then
474 echo >&2 "[$proj] error: make_repack_dir failed packed-refs extra line count sanity check"
475 exit 1
477 _lines="$_newlines"
478 optref="$(git rev-list -n 1 --all 2>/dev/null)" || :
479 if [ -n "$optref" ]; then
480 echo "$optref refs/tags/!" >>repack/packed-refs
481 _lines=$(( $_lines + 1 ))
482 echo "$optref" >repack/HEAD
483 else
484 cat HEAD >repack/HEAD
486 if [ $(LC_ALL=C wc -l <repack/packed-refs) -ne "$_lines" ]; then
487 echo >&2 "[$proj] error: make_repack_dir failed packed-refs line count sanity check"
488 exit 1
492 # Remove any crud that's been left behind by interrupted operations
493 # that did not clean up after themselves
494 remove_crud() {
495 # Remove any existing FETCH_HEAD
496 # There can only be a FETCH_HEAD if we've been fetching, not if we've been
497 # receiving pushes (those never create a FETCH_HEAD).
498 # And if we're fetching because we're a mirror, we know we're not fetching right
499 # now since jobd.pl never runs a project's fetch simultaneously with its gc.
500 # Therefore any existing FETCH_HEAD is junk. And it may be many megabytes if
501 # there were a lot of refs.
502 rm -f FETCH_HEAD
504 # Remove any stale ref locks
505 clear_stale_ref_locks
507 # remove any existing pack_is_complete_test or repack subdirectories
508 # If either exists when this function is called it's crud
509 rm -rf pack_is_complete_test repack objects/pack/repack
511 # Remove any stale pack remnants that are more than an hour old.
512 # Stale pack fragments are defined as any pack-<sha1>.ext where .ext is NOT
513 # .pack AND the corresponding .pack DOES NOT exist. A bunch of stale
514 # pack-<sha1>.idx files without their corresponding .pack files are worthless
515 # and just waste space. Normally there shouldn't be any remnants but actually
516 # this can happen when things are interrupted at just the wrong time.
517 # Note that the objects/pack directory is created by git init and should
518 # always exist.
519 find -L objects/pack -maxdepth 1 -type f -mmin +60 -name "pack-$octet20*.?*" -print |
520 LC_ALL=C sed -e 's/^objects\/pack\/pack-//; s/\..*$//' | LC_ALL=C sort -u |
521 while read packsha; do
522 ! [ -e "objects/pack/pack-$packsha.pack" ] || continue
523 rm -f "objects/pack/pack-$packsha".?*
524 done
526 # Remove any stale tmp reflogs files that are more than one hour old.
527 # Since they are created only while the pre-receive hook is running and
528 # all it does is process a bunch of refs passed to it on standard input
529 # it's inconceivable that it would ever take as much as an hour to run.
530 if [ -d reflogs ]; then
531 find -L reflogs -maxdepth 1 -type f -mmin +60 -name "tmp_*" -exec rm -f '{}' + || :
534 # Remove any stale object tmp_obj_* files that are more than 3 hours old.
535 # Really these files should only exist very briefly so there shouldn't be any
536 # but things happen that can end up leaving them behind.
537 find -L objects/$octet -maxdepth 1 -type f -mmin +180 -name "tmp_obj_?*" -exec rm -f '{}' + 2>/dev/null || :
539 # Remove any stale pack .keep files that are more than 12 hours old.
540 # We don't do anything to create any permanent pack .keep files, so they must
541 # be remnants from some failed push or something. Removing the .keep will
542 # allow the pack to be properly repacked.
543 find -L objects/pack -maxdepth 1 -type f -mmin +720 -name "pack-$octet20*.keep" -exec rm -f '{}' + || :
545 # Remove any stale tmp_pack_*, tmp_idx_*, tmp_bitmap_*, packtmp-* or .tmp-*-pack* files
546 # that are more than 12 hours old.
547 find -L objects/pack -maxdepth 1 -type f -mmin +720 \( \
548 -name "tmp_pack_?*" -o -name "tmp_idx_?*" -o -name "tmp_bitmap_?*" -o \
549 -name "packtmp-?*" -o -name ".tmp-?*-pack*" \
550 \) -exec rm -f '{}' + || :
552 # Remove any stale incoming-* object quarantine directories that are
553 # more than 12 hours old. These are new with Git >= 2.11.0.
554 find -L objects -maxdepth 1 -type d -name 'incoming-?*' -mmin +720 \
555 -exec rm -rf '{}' + || :
557 # Remove any stale shallow_* files that are more than 12 hours old.
558 # These can be left behind by Git >= 1.8.4.2 and < 2.0.0 when a client
559 # requests a shallow clone. Also discard stale .refs-temp* and
560 # .refs-new* files at the same time.
561 find -L . -maxdepth 1 -type f -mmin +720 \( \
562 -name "shallow_?*" -o -name ".refs-temp*" -o -name ".refs-new*" \
563 \) -exec rm -f '{}' + || :
565 # Remove any stale cmbnpcks-* dirs that are more than 12 hours old.
566 # These can be left behind by abnormal exits (e.g. power failure).
567 find -L . -maxdepth 1 -type d -mmin +720 -name "cmbnpcks-?*" \
568 -exec rm -rf '{}' + || :
570 # Remove any stale *.temp files in the objects area that are more than 12 hours old.
571 # This can be stale sha1.temp, or stale *.pack.temp so we kill all stale *.temp.
572 find -L objects -type f -mmin +720 -name "*.temp" -exec rm -f '{}' + || :
574 # Remove any stale *.lock files in the htmlcache area that might have been left
575 # behind after an abnormal exit during an attempt to update a cached file and
576 # are more than 1 hour old.
577 ! [ -d htmlcache ] || find -L htmlcache -type f -mmin +60 -name "*.lock" -exec rm -f '{}' + || :
579 # Remove any stale git-svn temp files that are more than 12 hours old.
580 # The git-svn process creates temp files with random 10 character names
581 # in the root of $GIT_DIR. Unfortunately they do not have a recognizable
582 # prefix, so we just have to kill any files with a 10-character name. We
583 # do this only for git-svn mirrors. All characters are chosen from
584 # [A-Za-z0-9_] so we can at least check that and fortunately the only
585 # collision is 'FETCH_HEAD' but that shouldn't matter.
586 # There may also be temp files with a Git_ prefix as well.
587 if [ -n "$svn_mirror" ]; then
588 _randchar='[A-Za-z0-9_]'
589 _randchar2="$_randchar$_randchar"
590 _randchar4="$_randchar2$_randchar2"
591 _randchar10="$_randchar4$_randchar4$_randchar2"
592 find -L . -maxdepth 1 -type f -mmin +720 -name "$_randchar10" -exec rm -f '{}' + || :
593 find -L . -maxdepth 1 -type f -mmin +720 -name "Git_*" -exec rm -f '{}' + || :
596 # Remove any stale fast_import_crash_<pid> files that are more than 3 days old.
597 if [ -n "$gfi_mirror" ]; then
598 find -L . -maxdepth 1 -type f -mmin +4320 -name "fast_import_crash_?*" -exec rm -f '{}' + || :
601 # Remove any stale core or *.core or core.* files that are more than 3 days old.
602 find -L . -maxdepth 1 -type f -mmin +4320 \( -name "core" -o -name "*.core" -o -name "core.*" \) \
603 -exec rm -f '{}' + || :
607 ## Garbage Collection Types
609 ## There are two kinds of possible garbage collection (gc) operations:
611 ## 1. A normal, full gc
612 ## 2. A "mini" gc
614 ## If the full garbage collection interval has expired (or gc has never been
615 ## run), then a normal, full gc will take place. Otherwise, a "mini" gc will
616 ## take place if the file .needsgc exists.
618 ## A "mini" gc is similar to "git gc --auto" in that it may not end up actually
619 ## doing anything unless the right conditions are present so it's not a burden
620 ## to run it often. If the file .needsgc exists, a "mini" gc will occur at
621 ## the next opportunity.
623 ## See the docs/technical/gc.txt and docs/technical/gc-mini.txt files for more
624 ## of the gory details of how garbage collection is performed.
626 ## Note, however, that the .nogc file suppresses ALL gc activity (normal or mini).
629 proj="${1%.git}"
630 shift
631 cd "$cfg_reporoot/$proj.git"
632 [ -d objects/pack ] || { rm -f gfi-packs; mkdir -p objects/pack; }
633 mirror_url="$(get_mirror_url)" || :
634 svn_mirror=
635 ! is_svn_mirror_url "$mirror_url" || svn_mirror=1
636 gfi_mirror=
637 if [ -f gfi-packs ] && [ -s gfi-packs ] && is_gfi_mirror_url "$mirror_url"; then
638 gfi_mirror=1
641 # If git config --bool --get girocco.redelta is explicitly false then automatic
642 # redelta when there are less than $var_redelta_threshold objects will be suppressed.
643 # On the other hand, if git config --get girocco.redelta is "always" then, on a full
644 # gc only, for the final repack, deltas will always be recomputed.
645 # This can be set on a per-project basis to avoid unusual pathological gc behavior.
646 # Setting this will hurt efficiency of the affected repository.
647 # Note that fast-import packs ALWAYS get new deltas regardless of this setting.
648 noreusedeltaopt="--no-reuse-delta"
649 [ "$(git config --bool --get girocco.redelta 2>/dev/null || :)" != "false" ] || noreusedeltaopt=
650 alwaysredelta=
651 [ "$(git config --get girocco.redelta 2>/dev/null || :)" != "always" ] || alwaysredelta=1
653 # Extract any -f or -F or --no-reuse-object or --no-reuse-delta options
654 # to be compatible with the old and new gc.sh versions and avoid ugly argument
655 # duplication in process lists at the same time
656 # Any options found will override the "girocco.redelta" setting
657 recompress=
658 idx=$#
659 while [ $idx -gt 0 ]; do
660 idx=$(( $idx - 1 ))
661 opt="$1"
662 shift
663 case "$opt" in
664 -f|--no-reuse-delta)
665 alwaysredelta=1
666 continue
668 -F|--no-reuse-object)
669 alwaysredelta=1
670 recompress=1
671 continue
673 -?*)
676 printf >&2 '%s\n' "bad non-option argument: $opt"
677 echo >&2 "(Did you perhaps intend to use a --xxx=yyy form?)"
678 exit 1
679 esac
680 [ -z "$opt" ] || set -- "$@" "$opt"
681 done
682 if [ -n "$alwaysredelta" ]; then
683 noreusedeltaopt="--no-reuse-delta"
684 [ -z "$recompress" ] || noreusedeltaopt="--no-reuse-object"
687 trap 'e=$?; rm -f .gc_in_progress; if [ $e != 0 ]; then echo "gc failed dir: $PWD" >&2; fi' EXIT
688 trap 'exit 130' INT
689 trap 'exit 143' TERM
691 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
692 datefmt='+%a, %d %b %Y %T %z'
694 isminigc=
695 if [ "${force_gc:-0}" = "0" ] && check_interval lastgc $cfg_min_gc_interval; then
696 if [ -e .needsgc ]; then
697 isminigc=1
698 else
699 progress "= [$proj] garbage check skip (last at $(config_get lastgc))"
700 exit 0
703 if [ -e .nogc ]; then
704 progress "x [$proj] garbage check disabled"
705 exit 0
707 if ! [ -e .nofetch ] && [ -e .clone_in_progress ] && ! [ -e .clone_failed ]; then
708 progress "x [$proj] garbage check disabled (clone in progress)"
709 exit 0
711 if [ -z "$isminigc" ] && [ -e .delaygc ] && [ -e .needsgc ]; then
712 # Eligible for a full gc but .delaygc is set so it would be skipped
713 # However .needsgc is also set so transform it into a mini instead
714 isminigc=1
715 progress "~ [$proj] garbage check delayed but checking mini because .needsgc"
718 if [ -n "$isminigc" ]; then
719 # Perform a "mini" gc
720 # Note that .delaygc is ignored here as that's only intended for full gc
721 lock_gc
722 rm -f .allowgc .needsgc
723 rm -f objects/pack/pack-*_[rful].keep
724 remove_crud
725 coalesce_reflogs
726 prune_reflogs
727 compact_reflogs
728 maintain_auto_gc_hack
729 generate_auto_gc_update
730 miniactive=
731 if [ -f .needspack ]; then
732 miniactive=1
733 progress "+ [$proj] mini garbage check ($(date))"
734 make_needs_pack
736 if [ -z "$cfg_delay_gfi_redelta" ] && [ -n "$gfi_mirror" ]; then
737 # $Girocco::Config::delay_gfi_redelta is false, force redeltification now
738 if [ -z "$miniactive" ]; then
739 miniactive=1
740 progress "+ [$proj] mini garbage check ($(date))"
742 repack_gfi_packs
744 if lotsa_loose_objects; then
745 if [ -z "$miniactive" ]; then
746 miniactive=1
747 progress "+ [$proj] mini garbage check ($(date))"
749 pack_loose_objects
751 # If there aren't at least 10 non-keep, non-bitmap, non-bndl packs then
752 # don't actually process them yet
753 lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl --quiet"
754 packcnt="$(list_packs --count $lpo objects/pack)" || :
755 if [ "${packcnt:-0}" -ge 10 ]; then
756 if [ -z "$miniactive" ]; then
757 miniactive=1
758 progress "+ [$proj] mini garbage check ($(date))"
760 # if we have at least 10 packs go ahead and pack all refs now too
761 git pack-refs --all --prune
762 if [ -n "$gfi_mirror" ]; then
763 repack_gfi_packs
764 packcnt="$(list_packs --count $lpo objects/pack)" || :
766 # if repack_gfi_packs dropped the pack count to < 10 don't combine
767 if [ "${packcnt:-0}" -ge 10 ]; then
768 combine_small_packs
769 combine_small_loose_packs
770 packcnt="$(list_packs --count $lpo objects/pack)" || :
772 # if we still have more than 10 packs trigger a full gc
773 if [ "${packcnt:-0}" -ge 10 ]; then
774 # We shouldn't be in a .delaygc state at this point, but if
775 # we are then nuke it because we really need a full gc now
776 rm -f .delaygc
777 git config --unset gitweb.lastgc
778 rm -f "$lockf"
779 git update-server-info # just in case
780 progress "- [$proj] mini garbage check triggering full gc too many packs ($(date))"
781 exit 0
784 rm -f "$lockf"
785 if [ -n "$miniactive" ]; then
786 git update-server-info
787 progress "- [$proj] mini garbage check ($(date))"
788 else
789 progress "= [$proj] mini garbage check nothing but crud removal to do ($(date))"
791 exit 0
794 # Avoid unnecessary garbage collections:
795 # 1. If lastreceive is set and is older than lastgc
796 # -AND-
797 # 2. We are not a fork (is_empty_alternates_file) -OR- lastparentgc is older than lastgc
799 # If lastgc is NOT set or lastreceive is NOT set we MUST run gc
800 # If we are a fork and lastparentgc is NOT set we MUST run gc
802 # If the repo is dirty after removing any crud we MUST run gc
804 gcstart="$(date "$datefmt")"
805 skipgc=
806 isfork=
807 is_empty_alternates_file objects/info/alternates || isfork=1
808 lastparentgcsecs=
809 [ -z "$isfork" ] || lastparentgcsecs="$(config_get_date_seconds lastparentgc)" || :
810 lastreceivesecs=
811 if lastreceivesecs="$(config_get_date_seconds lastreceive)" &&
812 [ "${force_gc:-0}" = "0" ] &&
813 lastgcsecs="$(config_get_date_seconds lastgc)" &&
814 [ $lastreceivesecs -lt $lastgcsecs ]; then
815 # We've run gc since we last received, so maybe we can skip,
816 # check if not fork or fork and lastparentgc < lastgc
817 if [ -n "$isfork" ]; then
818 if [ -n "$lastparentgcsecs" ] &&
819 [ $lastparentgcsecs -lt $lastgcsecs ]; then
820 # We've run gc since our parent ran gc so we can skip
821 skipgc=1
823 else
824 # We don't have any alternates (we're not a forK) so we can skip
825 skipgc=1
829 # Now check the simple signatures if $skipgc and decline to skip if they do not match
831 if [ -n "$skipgc" ]; then
832 # check the packs signature first, no previous sig => must run gc
833 phash="$(config_get girocco.gcsig.packs)" || :
834 [ -n "$phash" ] || skipgc=
835 if [ -n "$skipgc" ]; then
836 # a previous packs sig is available, check for a match
837 _chksig="$(list_packs -C objects/pack --exclude-no-idx --quiet . |
838 LC_ALL=C sort | sig_hash_stdin)" || :
839 [ "$phash" = "$_chksig" ] || skipgc=
842 if [ -n "$skipgc" ]; then
843 # check the refs signature second, no previous sig => must run gc
844 rhash="$(config_get girocco.gcsig.refs)" || :
845 [ -n "$rhash" ] || skipgc=
846 if [ -n "$skipgc" ]; then
847 # a previous refs sig is available, check for a match
848 v_compute_refs_sighash _chksig
849 [ "$rhash" = "$_chksig" ] || skipgc=
853 # Prevent any other simultaneous gc operations
854 lock_gc
856 # At this point, if .allowgc or .gc_failed exists, it's now crud to be removed
857 rm -f .allowgc .gc_failed
859 # Always get rid of crud
860 remove_crud
862 # Ideally we would do this in post-receive, but that would mean duplicating the
863 # logic so it's available in the chroot jail and that's highly undesirable
864 # Instead, since the first gc will be triggered immediately following the first
865 # push, we do the check here as it's quick and harmless if HEAD is already valid
866 check_and_set_head || :
868 # Always perform reflogs maintenance
869 coalesce_reflogs
870 prune_reflogs
871 compact_reflogs
873 # Always maintain auto gc hack
874 maintain_auto_gc_hack
875 generate_auto_gc_update
877 # Run 'git svn gc' now for svn mirrors
878 if [ -n "$svn_mirror" ]; then
879 git svn gc || :
882 # Skip the actual gc if .delaygc is set
883 if [ -e .delaygc ]; then
884 progress "x [$proj] garbage check delayed (except for crud removal)"
885 rm -f "$lockf"
886 exit 0
889 # Do not skip gc if the repo is dirty
890 if [ -n "$skipgc" ] && ! is_dirty; then
891 progress "= [$proj] garbage check nothing but crud removal to do ($(date))"
892 config_set lastgc "$gcstart"
893 rm -f "$lockf"
894 exit 0
897 bumptime=
898 if [ -n "$isfork" ] && [ -z "$lastparentgcsecs" ]; then
899 # set lastparentgc and then update gcstart to be at least 1 second later
900 config_set lastparentgc "$gcstart"
901 bumptime=1
903 if [ -z "$lastreceivesecs" ]; then
904 # set lastreceive and then update gcstart to be at least 1 second later
905 config_set lastreceive "$gcstart"
906 bumptime=1
908 if [ -n "$bumptime" ]; then
909 sleep 1
910 gcstart="$(date "$datefmt")"
913 progress "+ [$proj] garbage check ($(date))"
915 newdeltas=
916 [ -z "$alwaysredelta" ] || newdeltas="$noreusedeltaopt"
917 if [ -z "$newdeltas" ] && [ -n "$gfi_mirror" ]; then
918 if [ $(list_packs --exclude-no-idx --count objects/pack) -le \
919 $(list_packs --exclude-no-idx --count --quiet --only gfi-packs) ]; then
920 # Don't bother with repack_gfi_packs since everything's being repacked
921 newdeltas="--no-reuse-delta"
924 if [ -z "$newdeltas" ] && [ -n "$noreusedeltaopt" ] &&
925 [ $(list_packs --all --exclude-no-idx --count-objects objects/pack) -le $var_redelta_threshold ]; then
926 # There aren't enough objects to worry about so just redelta to get the best pack
927 newdeltas="--no-reuse-delta"
929 if [ -z "$newdeltas" ]; then
930 # Since we're not going to recompute deltas overall, we need to do the
931 # "mini" maintenance so that we can get more optimal deltas
932 [ -z "$noreusedeltaopt" ] || make_needs_pack
933 repack_gfi_packs
934 force_single_pack_redelta=
935 [ -n "$gfi_mirror" ] || [ -n "$svn_mirror" ] || force_single_pack_redelta=1
936 [ -z "$noreusedeltaopt" ] || combine_small_packs $force_single_pack_redelta
937 [ -z "$noreusedeltaopt" ] || combine_small_loose_packs $force_single_pack_redelta
941 ## Safe Pruning In Forks
943 ## We are about to perform garbage collection. We do NOT use the "git gc" or
944 ## the "git repack" commands directly as they do not provide enough control over
945 ## the fine details. However, we DO maintain a "gc.pid" file during our garbage
946 ## collection so that a simultaneous "git gc" by an administrator will be
947 ## blocked (and similarly we refuse to start garbage collection if we cannot
948 ## create the "gc.pid" file).
950 ## When we say "gc" in the below description we are referring to our "gc.sh"
951 ## script, NOT the "git gc" command.
953 ## If the project we are running garbage collection (gc) on has any forks we
954 ## must be careful not to remove any objects that while no longer referenced by
955 ## this project (the parent) are still referenced by one or more forks (the
956 ## children) otherwise the children will become corrupt and we can't abide
957 ## corrupt children.
959 ## One way to accomplish this is to simply hard-link all currently existing
960 ## loose objects and packs in the parent into all the children that refer to the
961 ## parent (via a line in their objects/info/alternates file) before beginning
962 ## the gc operation and then relying on a subsequent gc in the child to clean up
963 ## any excess objects/packs. We used to use this strategy but it's very
964 ## inefficient because:
966 ## 1. The disk space used by the old pack(s)/object(s) will not be reclaimed
967 ## until all children (and their children, if any) run gc by which time
968 ## it's quite possible the topmost parent will have run gc again and
969 ## hard-linked yet another old pack down to its children (not to mention
970 ## loose objects).
972 ## 2. When using the "-A" option with "git repack", any new objects in the
973 ## parent that are not referenced by children will continually get
974 ## exploded out of the hard-linked pack in the children whenever the
975 ## children run gc.
977 ## 3. To avoid suboptimal and/or unnecessarily many packs being hard-linked
978 ## into child forks, we must run the "mini" gc maintenance before we
979 ## perform the hard-linking into the children which provides yet another
980 ## source of inefficiency.
982 ## While we were still using the "-A" option to "git repack" (that was not
983 ## always the case) to guarantee we can access old ref values for long enough
984 ## to send out a meaningful mail.sh notification, another, more efficient,
985 ## option became available to prevent corruption of child forks that continue
986 ## to refer to objects that are no longer reachable from any ref in the parent.
988 ## The only things that need be copied (or hard-linked) into the child fork(s)
989 ## are those objects that have become unreachable from any ref in the parent.
991 ## When we were using the "git repack -A -d" + "git prune --expire=1.day.ago"
992 ## technique, the only objects that could ever be removed were loose objects
993 ## that "git prune" determined were expired. In that case, loose objects were
994 ## all that need be hard-linked down to child forks in order to avoid
995 ## corruption of any child fork(s).
997 ## The "git repack -A -d" + "git prune --expire=1.day.ago" + hard-linking loose
998 ## objects to child forks technique remains fundamentally sound from the
999 ## perspective of supporting simultaneous gc and push and keeping newly
1000 ## unreachable objects around long enough to be sure we can send out meaningful
1001 ## ref change notifications and never corrupting any child forks and never
1002 ## persisting the lifetime of large old packs containing mostly duplicate or
1003 ## unreachable objects as gc percolates through a project's entire fork tree.
1005 ## However, that technique suffers from one potential prodigious pitfall.
1007 ## Unreachable objects come flying out of their packs to splatter all over the
1008 ## objects subdirectories possibly creating a huge, inefficient mess.
1010 ## Often this is not an issue. Even with a lot of rebasing going on, usually
1011 ## the only objects that will splatter are some commits, trees and the odd blob
1012 ## here and there. Not enough to be overly concerned about.
1014 ## However, for the reppository that frequently experiences a lot of non-fast-
1015 ## forward updates and/or outright ref deletion, the number of objects suddenly
1016 ## popping out of their packs at "git repack -A -d" time can be overwhelming.
1018 ## To avoid this issue we now use a four phase pack creation strategy.
1019 ## This will result in creation of up to four packs (instead of at most one).
1021 ## I. A complete pack (with bitmaps if appropriate) gets created including
1022 ## only "reachable" objects from all refs/... refs plus HEAD. This will
1023 ## also serve as the virtual bundle for the repository.
1025 ## II. A pack of recently-became-unreachable objects and friends is created.
1026 ## (The "friends" are ref logs, linked working tree HEADs and indicies.)
1027 ## Because both the pre-receive and update.sh script record all ref
1028 ## changes we can easily choose the cut off point for "recently".
1029 ## It is only the fact we maintain those logs in the reflogs subdirectory
1030 ## that allows this step to be possible.
1032 ## III. If the repository has any forks with a non-zero length alternates file,
1033 ## yet another pack of "--keep-unreachable" objects is generated that will
1034 ## not actually be kept in the parent, but hard-linked into all the forks.
1036 ## IV. Finally, after running "git prune-packed", any remaining loose objects
1037 ## are migrated into a pack of their own.
1039 ## We then remove any non-.keep packs that existed before we started the
1040 ## process being careful to keep any same-pack pushes for the "Push Pack Redux"
1041 ## race condition (see docs/technical/gc.txt).
1043 ## By using "git pack-objects" directly we are able to accomplish this with
1044 ## very little additional effort.
1046 ## The packs produced by (III) are treated almost like ".keep" packs by child
1047 ## forks in that the objects in them are never repacked into any other
1048 ## "--keep-unreachable" packs (but they can migrate into phase I or II packs)
1049 ## and those phase III packs are then hard-linked into any grandchild forks.
1051 ## This avoids the space explosion that could occur if each fork level ended
1052 ## up duplicating the "--keep-unreachable" pack space by repacking those
1053 ## objects (essentially breaking the hard-link to the single copy of those
1054 ## objects).
1056 ## While it is true that each level of forks could potentially add yet another
1057 ## phase III pack to be hard-linked down to its children, such packs will only
1058 ## include unreachable objects not already in any phase III packs that were
1059 ## received from the parent.
1061 ## The space for the phase III packs will not be reclaimed until the gc
1062 ## finishes percolating through the entire "fork tree" of a project.
1064 ## This is not much different than the "git repack -A -d" situation where
1065 ## all the loose objects are hard-linked down into child forks. In that
1066 ## case forks that actually need any of those objects could gradually reduce
1067 ## the number of objects hard-linked into deeper fork levels.
1069 ## The difference with a phase III "--keep-unreachable" pack is that there
1070 ## cannot be any gradual reduction like that since it would require repacking
1071 ## the pack and breaking the hard-link thereby increasing storage space. The
1072 ## storage will instead always be reclaimed all at once when all of the
1073 ## projects in the "fork tree" complete their gc.
1075 ## However, the belief is that the huge space win by having all the
1076 ## unreachable objects packed up together far eclipses (when many objects are
1077 ## involved, the single-pack version can end up using 1/20th or less of the
1078 ## disk space compared to having them all as loose objects) any brief minor
1079 ## space savings that might occur under the "git repack -A -d" loose object
1080 ## system prior to the gc collection completing for all the projects in the
1081 ## "fork tree".
1085 ## utility functions
1088 make_packs_ugw() {
1089 find -L "$1" -maxdepth 1 -type f ! -perm -ug+w \
1090 -name "pack-$octet20*.pack" -exec chmod ug+w '{}' + || :
1091 } 2>/dev/null
1093 get_index_tree() {
1094 if [ -s "$1" ]; then
1095 GIT_INDEX_FILE="$1"
1096 export GIT_INDEX_FILE
1097 git write-tree 2>/dev/null || :
1098 unset GIT_INDEX_FILE
1102 get_detached_head() {
1103 if [ -s "$1" ] && read -r _head <"$1" 2>/dev/null; then
1104 case "$_head" in $octet20*)
1105 echo "$_head"
1106 esac
1110 # single argument must be a top-level GIT_DIR
1111 # output will be (one per line, zero or more lines) full hash
1112 # values (i.e. 40 or more hex digits) from possible detached head
1113 # "single level" refs (e.g. "FETCH_HEAD" "MERGE_HEAD" etc.) from
1114 # files in the specified directory that have a "suitable" name and
1115 # have a length >= 40 and <= 1000 where EVERY line in a file MUST
1116 # match a 40 digit (or longer) hex string or that file will be ignored.
1117 # In a nod to Git each line first has any tab and anything following truncated.
1118 # NO sorting NOR "uniq"ing NOR "--batch-check"ing is performed on the output.
1119 # Also note that "HEAD" is NOT EXCLUDED and if detached will be output!
1120 get_detached_friends() {
1121 if [ -n "$1" ] && [ -d "$1" ]; then
1122 LC_ALL=C find -H "$1" -maxdepth 1 -name '[A-Za-z]*[A-Za-z0-9]' \
1123 -type f -size +39c -size -1001c -exec awk '
1124 BEGIN {exit}
1125 function process(fn, fnt, hashes, hcnt, fl, hi) {
1126 fnt = fn; sub(/^.*\//, "", fnt)
1127 if (length(fnt) > 31 || fnt !~ /^[A-Za-z][A-Za-z0-9_-]*[A-Za-z0-9]$/) return;
1128 hcnt = 0;
1129 while (getline fl < fn) {
1130 sub(/\t.*$/, "", fl)
1131 if (length(fl) < 40 || fl !~ /^[0-9a-fA-F][0-9a-fA-F]*$/) {hcnt = 0; break;}
1132 hashes[++hcnt] = fl;
1134 close(fn)
1135 for (hi=1;hi<=hcnt;++hi) print tolower(hashes[hi]);
1137 END {for (idx=1;idx<ARGC;++idx) process(ARGV[idx])}
1138 ' '{}' '+' 2>/dev/null || :
1142 # get_worktrees_friends
1143 # single argument is a (possibly relative) path to a "worktrees" subdir
1144 # if omitted it defaults to "worktrees"
1145 # any "friends" found in there are output to stdout
1146 get_worktrees_friends() {
1147 if [ -d "${1:-worktrees}" ]; then
1148 find -L "${1:-worktrees}" -mindepth 2 -maxdepth 2 -name HEAD -type f -print |
1149 while read -r _lwth; do
1150 get_detached_head "$_lwth"
1151 get_detached_friends "${_lwth%HEAD}"
1152 get_index_tree "${_lwth%HEAD}index"
1153 done
1157 # compute_extra_reachables
1158 # create lines suitable for a packed-refs file mentioning all the
1159 # other refs we might like to keep.
1160 # the current directory MUST be set to the repository's --git-dir
1161 # the following are included:
1162 # * refs mentioned in repack/packed-refs.extra (if it exists)
1163 # * refs mentioned in reflogs/... files
1164 # * tree(s) created from index file(s)
1165 # * detached linked working tree heads
1166 # Resulting objects are tested for existence and uniqified then output
1167 # one per line under a refs/z* namespace
1168 compute_extra_reachables() {
1170 if [ -s repack/packed-refs.extra ]; then
1171 LC_ALL=C sed <repack/packed-refs.extra -n \
1172 -e 's/^\([0-9A-Fa-f][0-9A-Fa-f]*\).*$/\1/p' \
1173 -e 's/^\^\([0-9A-Fa-f][0-9A-Fa-f]*\).*$/\1/p'
1175 digits8='[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]'
1176 find -L reflogs -mindepth 1 -maxdepth 1 -type f -name "$digits8*" -exec gzip -c -d -f '{}' + |
1177 LC_ALL=C awk '{print $2; print $3}'
1178 get_detached_friends .
1179 ! [ -f index ] || get_index_tree index
1180 get_worktrees_friends
1182 is_git_dir private &&
1183 [ "$(cd objects && pwd -P)" = "$(cd private/objects && pwd -P)" ] &&
1184 [ "$(cd refs && pwd -P)" != "$(cd private/refs && pwd -P)" ]
1185 then
1186 git --git-dir=private show-ref --head --hash 2>/dev/null || :
1187 get_detached_friends private
1188 ! [ -f private/index ] || get_index_tree private/index
1189 get_worktrees_friends private/worktrees
1191 } | LC_ALL=C sort -u |
1192 git cat-file ${var_have_git_260:+--buffer} --batch-check"${var_have_git_185:+=%(objectname)}" |
1193 LC_ALL=C awk '!/missing/ {num++; print $1 " " "refs/" substr("zzzzzzzzzzzz", 1, length(num)) "/" num}'
1196 # cat_all_ref_logs
1197 # the current directory MUST be set to the repository's --git-dir
1198 # first the arguments (which must reference readable files) are cat'd if any
1199 # then the path names of all ref log files are sorted and cat'd in that order
1200 # included in output:
1201 # * the contents of the pathnames given as arguments (if any) in order
1202 # * any existing files in the logs subdirectory and any existing worktree HEAD logs
1203 # but always in sorted filename order
1204 # obviously any missing directories are silently skipped
1205 cat_all_ref_logs() {
1206 [ $# -le 0 ] || LC_ALL=C cat -- "$@"
1208 ! [ -d logs ] || find -L logs -name '.*' -prune -o -name '*.' -prune -o -type f -size +0c -print
1209 if [ -d worktrees ]; then
1210 find -L worktrees -mindepth 2 -maxdepth 2 -name HEAD -type f -print |
1211 while read -r _lwth; do
1213 [ -d "${_lwth%HEAD}logs" ] &&
1214 [ -f "${_lwth%HEAD}logs/HEAD" ]
1215 then
1216 printf '%s\n' "${_lwth%HEAD}logs/HEAD"
1218 done
1220 } | LC_ALL=C sort | LC_ALL=C sed 's/[^A-Za-z0-9._/-]/\\&/g' | xargs env LC_ALL=C cat --
1224 ## main gc logic
1227 # Everything else is more efficient if we do this first
1228 # The "--prune" option is the default since v1.5.0 but it serves as "documentation" here
1229 git pack-refs --all --prune
1230 [ -e packed-refs ] || >>packed-refs # should never happen...
1232 # We didn't used to do anything about rerere or worktrees but we're
1233 # trying to make nice with linked working trees these days :)
1234 # Maybe even non-bare repositories too, but *shush* about those ;)
1236 # If we have a worktrees subdir and are running Git >= 2.5.0, prune worktrees
1237 # By doing this first we may save some work if any worktrees are pruned
1238 # Git will remove the worktrees subdir entirely when the last worktree is pruned
1239 if [ -n "$var_have_git_250" ] && [ -d worktrees ]; then
1240 # The value "3.months.ago" is hard-coded into gc.c rather than
1241 # having the default be in worktree.c so we must provide it if
1242 # we get nothing out of the gc.worktreePruneExpire config item
1243 # Prior to Git v2.6.0 the config item was gc.pruneworktreesexpire
1244 # however we just always use the newer name no matter what Git version
1245 expiry="$(git config --get gc.worktreePruneExpire 2>/dev/null)" || :
1246 eval git worktree prune --expire '"${expiry:-3.months.ago}"' "${quiet:+>/dev/null 2>&1}" || :
1249 # If we have a logs directory or a worktrees directory expire the ref logs now
1250 # Note that Git itself does not use either --rewrite or --updateref, so neither do we
1251 ! [ -d logs ] && ! [ -d worktrees ] || eval git reflog expire --all "${quiet:+>/dev/null 2>&1}" || :
1253 # git rerere does it right and handles its own default/config'd expiration values
1254 ! [ -d rr-cache ] || eval git rerere gc "${quiet:+>/dev/null 2>&1}" || :
1256 make_repack_dir
1257 ! [ -e .gc_failed ] || exit 1
1258 rm -f .gc_in_progress # make sure
1259 touch .gc_in_progress # it's truly fresh
1260 rm -f bundles/* objects/pack/pack-*.bndl
1261 # These only exist for a brief time before the packs loose their _f suffix
1262 # "Push Pack Redux" does not apply to these since they were only ever present with _f
1263 rm -f objects/pack/pack-*_f.keep
1264 # This is perhaps a bit aggressive in that if we're suffering from "Push Pack Redux"
1265 # and somehow we get run again immediately after the run where "Push Pack Redux" happened
1266 # and we have garbage collection forced, there's just the barest, almost negligible,
1267 # possibility that the "Push Pack Redux" ref updates _still_ have not happened and we
1268 # should not be removing _r .keep files. None of the normal Girocco processing can
1269 # cause this. The second run of this script would have to use the force gc option
1270 # for it to even be possible in the first place. What's much more likely is that
1271 # the initial run of this script was somehow interrupted in the middle before it
1272 # could get rid of the _r .keep file itself in which case it's better to get rid of
1273 # it now to avoid keeping something around that would perturb our nice and neat gc
1274 rm -f objects/pack/pack-*_r.keep
1275 # We will add .keep files for _u and _l packs if and when we run phase III
1276 # Otherwise they need to not have any .keep files during phases I and II
1277 rm -f objects/pack/pack-*_[ul].keep
1279 # We need to make sure that any non-Girocco (barely tolerated) Git object creation
1280 # activity will be able to "freshen" the pack containing a pre-existing object
1281 # that's being written. This really should not be necessary as the pre-receive
1282 # hook should make sure this takes place for any incoming pushes.
1283 # However, do it here anyway just in case.
1284 make_packs_ugw objects/pack
1286 # This is only effective with Git v2.3.5 and later and it will only matter when
1287 # we are using one of the "internal_rev_list" modes of pack-objects
1288 # (the combine-packs.sh script never uses any of those modes)
1289 # The "git repack" and "git prune" commands always set this internally themselves
1290 # It makes no difference if there's no repository corruption
1291 GIT_REF_PARANOIA=1 && export GIT_REF_PARANOIA
1293 # All of the options we might want to use with pack-objects were supported
1294 # at some point prior to Git version v1.6.6 which is the minimum version that
1295 # Girocco now requires. Except for one (--use-bitmap-index). Several of them
1296 # are "boiler plate" options we always want to use so we bundle them up here.
1297 pkopt="--delta-base-offset --keep-true-parents --non-empty --all-progress-implied"
1298 # We want to use --include-tag, but before Git v2.10.1 it would leave out
1299 # "middle" tags (e.g. a tag of a tag of a commit would omit the tagged tag)
1300 # See http://repo.or.cz/git.git/b773ddea2cd3b08c for details
1301 # ("pack-objects: walk tag chains for --include-tag", 2016-09-07, v2.10.1)
1302 # This is not a free check as it matches all refs against refs/tags/ then
1303 # peels all the annotated tags and checks for inclusion. The situation in
1304 # which it would add a tag that was not already included by a reachability
1305 # trace that included tag starting points can only occur if a new tag gets
1306 # pushed during gc pointing to something that would have been packed anyway.
1307 # But, it could happen and, really, compared to gc as a whole it's not that
1308 # expensive to perform (provided we do not get an unconnected pack).
1309 [ -z "$var_have_git_2101" ] || pkopt="$pkopt --include-tag"
1310 pkopt="$pkopt ${quiet:---progress} $packopts"
1312 # The git pack-objects command only supports bitmaps if all objects are being
1313 # packed (the "--all" option) and the "--stdout" option is NOT being used.
1314 # Additionally, while packing, if any encountered reachable objects are
1315 # determined to be "not wanted" then no bitmap index will be written anyway.
1316 # While it is theoretically possible that a project with a non-empty alternates
1317 # file ends up packing all objects (because it does not actually use any of the
1318 # objects found in the alternates), it's very unlikely. And, in the unlikely
1319 # event that did occur, clients would see a message about only using one bitmap
1320 # because Git can only use one bitmap at a time and at least one of the
1321 # alternates is bound to have a bitmap. Therefore if we see a non-empty
1322 # alternates file, we disable writing bitmaps which avoids the warning and any
1323 # possibility of a client warning as well. Also if we are running anything
1324 # before Git v2.1.0 (the effective version for repack.writeBitmaps=true) then
1325 # we also always disable bitmap writing.
1326 wbmopt=
1327 [ -z "$var_have_git_210" ] || wbmopt="--write-bitmap-index"
1328 # More recent versions of pack-objects have optimizations when not using the
1329 # --local option. If we do not have any alternates it's a pointless option.
1330 # If we do have alternates we need to skip writing a bitmap and we cannot
1331 # have a bundle since it must contain all objects.
1332 if [ -n "$isfork" ]; then
1333 lclopt="--local"
1334 wbmopt=
1335 makebndl=
1336 else
1337 lclopt=
1338 makebndl=1
1342 ## Phase I
1345 wbmstr=
1346 [ -n "$wbmopt" ] || wbmstr=" (bitmaps disabled)"
1347 progress "~ [$proj] running primary full gc pack-objects$wbmstr ($(date))"
1349 gotforks=
1350 ! has_forks_with_alternates "$proj" || gotforks=1
1352 # To avoid "Push Pack Redux" (see docs/technical/gc.txt), after collecting the
1353 # initial preexisting non-keep pack list, we rename them so that an incoming push
1354 # pack cannot possibly experience a pack name collision. Git does not require
1355 # use of the "default" pack names, simply that the proper extensions are used.
1356 # We rename to insert an "_r" just before the extension to avoid "Push Pack Redux"
1357 # name collisions. Later on we may create an "unreachable" pack for hard-linking
1358 # down into forks and it will have an "_u" inserted just before its extension.
1359 packlist="$(list_packs -C objects/pack --all --exclude-no-idx --exclude-keep --quiet .)" || :
1360 oldpacks=
1361 for oldpack in $packlist; do
1362 oldpack="${oldpack%.pack}"
1363 [ -f "objects/pack/$oldpack.pack" ] || {
1364 echo >&2 "[$proj] unable to list old pack files"
1365 exit 1
1367 case "$oldpack" in pre-auto-gc-[12])
1368 # we never disturb pre-auto-gc-1 or pre-auto-gc-2 packs
1369 continue
1370 esac
1371 oldpackhex="${oldpack#pack-}"
1372 if [ "${oldpackhex#*[!0-9a-fA-F]}" != "$oldpackhex" ]; then
1373 # names not exclusively hexadecimal do not need renaming
1374 case "$oldpack" in
1375 pack-$octet20*_l)
1376 # _l packs are treated like still-unpacked loose objects
1377 continue;;
1378 *_f)
1379 # _f packs can only be left over from a previously interrupted gc;
1380 # they need to be renamed to _r now so they're not confused with
1381 # any freshly generated "final" packs (and we already removed
1382 # any pre-existing *_f.keep files so we're good to go)
1385 oldpacks="${oldpacks:+$oldpacks }$oldpack"
1386 continue;;
1387 esac
1389 rename_pack "objects/pack/$oldpack" "objects/pack/${oldpack%_f}_r" || {
1390 echo >&2 "[$proj] unable to rename old pack files"
1391 exit 1
1393 # If the oldpack has a .keep now it means a "Push Pack Redux" is actually
1394 # in progress at this moment and we need to .keep the renamed pack,
1395 # otherwise no "Push Pack Redux" has started yet or it has already finished.
1396 # In either case we're okay because if it's just finished then all ref
1397 # changes have already been made so we don't need a .keep and we will
1398 # see the ref changes and grab all the objects via a reachability trace.
1399 # If it hasn't started yet that's okay because we're done moving that
1400 # name so a complete pack will appear under the old name that we'll
1401 # leave alone.
1402 if [ -f "objects/pack/$oldpack.keep" ]; then
1403 echo "Push Pack Redux" >"objects/pack/${oldpack%_f}_r.keep"
1404 else
1405 oldpacks="${oldpacks:+$oldpacks }${oldpack%_f}_r"
1407 done
1409 # We wish to keep deltas from our last full pack so if we're not redeltaing
1410 # then make sure the .pack associated with the .bitmap has a newer mod time
1411 # (If there is no .bitmap then touch the pack with the most objects instead.)
1412 if [ -z "$newdeltas" ]; then
1413 bmpack="$(list_packs --exclude-no-bitmap --exclude-no-idx --max-matches 1 objects/pack)"
1414 [ -n "$bmpack" ] || bmpack="$(list_packs --exclude-no-idx --max-matches 1 --object-limit -1 --include-boundary objects/pack)"
1415 if [ -n "$bmpack" ] && [ -f "$bmpack" ] && [ -s "$bmpack" ]; then
1416 sleep 1
1417 touch -c "$bmpack" 2>/dev/null || :
1418 # We must touch .gc_in_progress here to avoid $bmpack looking
1419 # like it's been "freshened" when redundant packs are removed
1420 # It's okay if they have the same mod time, but POSIX does not
1421 # guarantee an ordering for the "touching" that occurs which is
1422 # why this must be a separate command but needs no "sleep 1"
1423 touch .gc_in_progress
1427 # Now we need to make sure that any "freshening" that takes place will actually
1428 # result in a "newer" modification time than the .gc_in_progress file now has
1429 sleep 1
1431 # We run git pack-objects from the repack subdirectory so we can force
1432 # optimized packs to be generated even for repositories that do not have any
1433 # tagged commits
1434 packs="$(git --git-dir=repack pack-objects </dev/null \
1435 $pkopt --all $newdeltas $lclopt ${wbmopt:---honor-pack-keep} "$@" repack/alt/pack/pack)"
1436 v_cnt packcnt $packs
1437 [ $packcnt -eq 1 ] || makebndl=
1440 ## Phase II
1443 progress "~ [$proj] running supplementary gc pack-objects ($(date))"
1445 # Add the "supplementary" refs
1446 compute_extra_reachables >>repack/packed-refs
1448 # Now calculate the final repack signature hash
1449 # Since we do _allow_ worktrees and we do _allow_ ref logs,
1450 # the ref logs of such must be accounted for here as well
1451 _rpksig="$(cat_all_ref_logs repack/packed-refs | sig_hash_stdin)"
1452 echo "$_rpksig" >repack/repack.sighash
1454 # Subtract the primary refs
1455 GIT_ALTERNATE_OBJECT_DIRECTORIES="$PWD/repack/alt"
1456 export GIT_ALTERNATE_OBJECT_DIRECTORIES
1458 # For this one we MUST use --local and MUST NOT use --write-bitmap-index
1459 # However, if there is a "logs" subdirectory we need to use --reflog
1460 # We do add it, just in case, if the linked working trees dir is present
1461 # We do not add --indexed-objects as that requires v2.2.0 and it's unclear
1462 # if it properly includes linked working tree index files or not. The
1463 # above compute_extra_reachables has already included all index trees (thereby
1464 # providing proper --indexed-objects support for all Git versions) making the
1465 # option completely unnecessary.
1466 rflopt=
1467 ! [ -d logs ] && ! [ -d worktrees ] || rflopt=--reflog
1468 spacks="$(git --git-dir=repack pack-objects </dev/null \
1469 $pkopt --honor-pack-keep --all $rflopt $newdeltas --local "$@" repack/alt/pack/pack)"
1472 ## Phase III
1475 # There's nothing to do for Phase III unless we have forks that refer to our
1476 # project from their alternates file
1477 hlpacks=
1478 upacks=
1479 if [ -n "$gotforks" ]; then
1481 progress "~ [$proj] running keep-unreachable gc pack-objects for forks ($(date))"
1483 # If we are a fork, any pre-existing _u packs need to have a .keep
1484 # for this phase and be added to the hlpacks list otherwise (we are
1485 # not a fork) pre-existing _u packs are anomalies to be treated like
1486 # regular non-_u packs
1487 if [ -n "$isfork" ]; then
1488 for upack in $(find -L objects/pack -mindepth 1 -maxdepth 1 -name "pack-$octet20*_[ul].pack" -print); do
1489 upack="${upack%.pack}"
1490 [ -e "$upack.keep" ] || echo "unreachable" >"$upack.keep"
1491 case "$upack" in *_l);;*)
1492 hlpacks="${hlpacks:+$hlpacks }${upack#objects/pack/pack-}"
1493 esac
1494 done
1496 # Using either --no-reuse-delta or --no-reuse-object together with the
1497 # --keep-unreachable option is a very, very, very bad idea when good
1498 # packs are the desired outcome. If newdeltas are being generated
1499 # then we pack to a temp name, and use combine-packs.sh to get a better
1500 # pack as the result to avoid making a bad --keep-unreachable pack
1501 pfx=
1502 [ -z "$newdeltas" ] || pfx="ku"
1503 upacks="$(git --git-dir=repack pack-objects </dev/null \
1504 $pkopt --honor-pack-keep --all $rflopt --keep-unreachable --local "$@" repack/alt/pack/${pfx}pack)"
1505 if [ -n "$upacks" ] && [ -n "$newdeltas" ]; then
1506 progress "~ [$proj] rebuilding keep-unreachable pack deltas"
1507 oldupacks="$upacks"
1508 upacks="$(
1509 printf "repack/alt/pack/${pfx}pack-%s.pack\n" $oldupacks |
1510 run_combine_packs --names --weak-naming --non-empty --all-progress-implied ${quiet:---progress} \
1511 $packopts $newdeltas "$@" repack/alt/pack/pack)"
1512 eval rm -f "$(printf \""repack/alt/pack/${pfx}pack-%s.*"\"" " $oldupacks)"
1514 for upack in $upacks; do
1515 rename_pack "repack/alt/pack/pack-$upack" "repack/alt/pack/pack-${upack}_u"
1516 done
1517 rm -f objects/pack/pack-*_[ul].keep
1518 [ -z "$hlpacks" ] && [ -z "$upacks" ] ||
1519 progress "~ [$proj] hard-linking keep-unreachable pack(s) into immediate child forks"
1521 # We have to update the lastparentgc time in the child forks even if they do not get any
1522 # new "unreachable packs" because they need to run gc just in case the parent now has some
1523 # objects that used to only be in the child so they can be removed from the child.
1524 # For example, a "patch" might be developed first in a fork and then later accepted into
1525 # the parent in which case the objects making up the patch in the child fork are now
1526 # redundant (since they're now in the parent as well) and need to be removed from the
1527 # child fork which can only happen if the child fork runs gc.
1528 lastparentgc="$(date "$datefmt")"
1530 # It is enough to copy objects just one level down and get_repo_list
1531 # takes a regular expression (which is automatically prefixed with '^')
1532 # so we can easily match forks exactly one level down from this project
1533 forkdir="$proj"
1534 get_repo_list "$forkdir/[^/:][^/:]*:" |
1535 while read fork; do
1536 # Ignore forks that do not exist or are symbolic links
1537 ! [ -L "$cfg_reporoot/$fork.git" ] && [ -d "$cfg_reporoot/$fork.git" ] ||
1538 continue
1539 # Or have an empty alternates file
1540 ! is_empty_alternates_file "$cfg_reporoot/$fork.git/objects/info/alternates" ||
1541 continue
1542 runupdate=
1543 # Match hlpacks in parent project if any
1544 if [ -n "$hlpacks" ]; then
1545 mkdir -p "$cfg_reporoot/$fork.git/objects/pack"
1546 eval ln -f "$(printf '"objects/pack/pack-%s.pack" ' $hlpacks)" \
1547 "$(printf '"objects/pack/pack-%s.idx" ' $hlpacks)" \
1548 '"$cfg_reporoot/$fork.git/objects/pack/"'
1549 runupdate=1
1551 # Match upacks in repack/alt area if any
1552 if [ -n "$upacks" ]; then
1553 mkdir -p "$cfg_reporoot/$fork.git/objects/pack"
1554 eval ln -f "$(printf '"repack/alt/pack/pack-%s_u.pack" ' $upacks)" \
1555 "$(printf '"repack/alt/pack/pack-%s_u.idx" ' $upacks)" \
1556 '"$cfg_reporoot/$fork.git/objects/pack/"'
1557 runupdate=1
1559 if ! [ -e "$cfg_reporoot/$fork.git/.needsgc" ]; then
1560 # Trigger a mini gc in the fork if it now has too many packs
1561 fpackcnt="$(list_packs --quiet --count --exclude-no-idx --exclude-keep "$cfg_reporoot/$fork.git/objects/pack")" || :
1562 if [ -n "$fpackcnt" ] && [ "$fpackcnt" -ge 20 ]; then
1563 >"$cfg_reporoot/$fork.git/.needsgc"
1566 [ -z "$runupdate" ] || git --git-dir="$cfg_reporoot/$fork.git" update-server-info
1567 # Update the fork's lastparentgc date (must be more recent than $gcstart)
1568 git --git-dir="$cfg_reporoot/$fork.git" config gitweb.lastparentgc "$lastparentgc"
1569 done
1572 # Now move any primary/supplementary packs back into objects/pack
1573 # then drop any "unfreshened" redundant packs and clear repack/alt
1575 # First make sure the primary pack(s) have the most recent mod time
1576 if [ -n "$packs" ]; then
1577 [ -z "$spacks" ] || sleep 1
1578 printf 'repack/alt/pack/pack-%s.pack\n' $packs | xargs touch -c 2>/dev/null || :
1581 # Move the packs into place but with a _f suffix and a .keep file for now
1582 for pack in $packs $spacks; do
1583 rename_pack "repack/alt/pack/pack-$pack" "objects/pack/pack-${pack}_f"
1584 [ -e "objects/pack/pack-${pack}_f.keep" ] ||
1585 echo "final" >"objects/pack/pack-${pack}_f.keep"
1586 done
1588 # It's possible that one of the $oldpacks had a .bitmap, got renamed (along
1589 # with its .bitmap) and then got "freshened" causing us to not remove it
1590 # However, if $wbmopt is set we most likely now have TWO .bitmap packs!
1591 # This can produce ugly warnings we don't want and possibly get the wrong
1592 # bitmap used since only one .bitmap file can ever be used by Git.
1593 # If this has happened, the .bitmap we want to discard will always have
1594 # an _r suffix so we can just zap any such now since it will leave the pack.
1595 [ -z "$wbmopt" ] || rm -f objects/pack/pack-*_r.bitmap || :
1597 # Remove the redundant packs that have not since been "freshened"
1598 # This does not completely eliminate the race condition window (Girocco's own
1599 # activites -- gc/fetch/receive are immune to the race) but it substantially
1600 # shrinks it down to just the time after the find but before the following rm
1601 >repack/oldpacks
1602 [ -z "$oldpacks" ] ||
1603 printf 'objects/pack/%s.pack\n' $oldpacks |
1604 LC_ALL=C sort >repack/oldpacks
1605 find -L objects/pack -maxdepth 1 -type f -name "pack-$octet20*.pack" -newer .gc_in_progress -print |
1606 LC_ALL=C sort >repack/freshened
1607 deadpacks="$(LC_ALL=C join -v 1 repack/oldpacks repack/freshened | LC_ALL=C sed 's/\.pack$//')"
1608 [ -z "$deadpacks" ] ||
1609 eval echo "$(printf '"%s".* ' $deadpacks)" | xargs rm -f || :
1611 # No need for this anymore
1612 rm -rf repack/alt objects/pack/repack
1613 unset GIT_ALTERNATE_OBJECT_DIRECTORIES
1616 ## Phase IV
1619 progress "~ [$proj] running gc prune-packed"
1621 # We do not want the redundant packs or any new "--keep-unreachable" pack(s) to be
1622 # present while running prune-packed. We try to guarantee that any loose object
1623 # (or any object present in a pack with an _l suffix which was created by mini gc)
1624 # that's unreachable persists for at least one $Girocco::Config::min_gc_interval
1625 # (not withstanding administrator interference to force earlier gc to occur).
1626 # If we were to include the redundant/keep-unreachable pack(s) when running
1627 # prune-packed and a loose unreachable object happened to be duplicated in one
1628 # of them we would end up removing it too soon and void our guarantee.
1629 git prune-packed $quiet
1631 progress "~ [$proj] running loose objects gc pack-objects ($(date))"
1633 # Although Git v2.10.0 and later support a --pack-loose-unreachable option,
1634 # we MUST NOT use it for these reasons:
1635 # 1) We're not interested in expensive "unreachable" at this point, only "loose"
1636 # 2) It produces simply horrid packs about 3.8x times larger than they should be
1637 # 3) We don't require anything more than Git v1.6.6
1638 # The only way we could see any _o pack files at this point is if one got
1639 # "freshened" while we were running gc. If that happens then it gets to live on
1640 # until the next full gc and we need to include it in the loose repack here.
1641 lpacks="$(list_packs --exclude-no-idx --exclude-no-sfx _l --exclude-no-sfx _o --quiet objects/pack |
1642 run_combine_packs --replace --names --loose --weak-naming --non-empty --honor-pack-keep \
1643 --all-progress-implied ${quiet:---progress} $packopts $newdeltas "$@")"
1645 if [ -n "$lpacks" ]; then
1646 # Make sure any primary pack(s) have a more recent mod time than "unreachable" objects packs
1647 if [ -n "$packs" ]; then
1648 sleep 1
1649 printf 'objects/pack/pack-%s_f.pack\n' $packs | xargs touch -c 2>/dev/null || :
1651 # We need to identify these packs later so we don't combine_packs them
1652 for objpack in $lpacks; do
1653 rename_pack "objects/pack/pack-$objpack" "objects/pack/pack-${objpack}_o" || :
1654 done
1657 # Polish up the final packs now
1658 rm -f objects/pack/pack-*_f.keep
1659 for pack in $packs $spacks; do
1660 rename_pack "objects/pack/pack-${pack}_f" "objects/pack/pack-$pack"
1661 done
1663 if [ -n "$lpacks" ]; then
1664 # Finally zap the corresponding loose objects
1665 progress "~ [$proj] running packed loose objects gc prune-packed"
1666 git prune-packed $quiet
1669 ! [ -e .gc_failed ] || exit 1
1670 # These, if they exist, are now meaningless and need to be removed
1671 rm -f gfi-packs .needsgc .needspack .needspackgc
1673 # Make sure this stays up to date
1674 git update-server-info
1676 # We must make loose objects group writable so that they
1677 # can be freshened by other pushers. Technically we need only do this for
1678 # push projects but to enable mirror projects to be more easily converted to
1679 # push projects, we go ahead and do it for all projects.
1680 # By the time we get here we really shouldn't have any of these, but just in case.
1681 { find -L objects/$octet -type f -name "$octet19*" -exec chmod ug+w '{}' + || :; } 2>/dev/null
1683 # darcs mirrors have a xxx.log file that will grow endlessly
1684 # if this is a mirror and the file exists, shorten it to 10000 lines
1685 # also take this opportunity to optimize the darcs repo
1686 if ! [ -e .nofetch ] && [ -n "$cfg_mirror" ]; then
1687 url="$(config_get baseurl)" || :
1688 case "$url" in darcs://* | darcs+http://* | darcs+https://*)
1689 if [ -n "$cfg_mirror_darcs" ]; then
1690 url="${url%/}"
1691 basedarcs="$(basename "${url#darcs*:/}")"
1692 if [ -f "$basedarcs.log" ]; then
1693 tail -n 10000 "$basedarcs.log" >"$basedarcs.log.$$"
1694 mv -f "$basedarcs.log.$$" "$basedarcs.log"
1696 if [ -d "$basedarcs.darcs" ]; then
1698 cd "$basedarcs.darcs"
1699 # without show_progress suppress non-error output
1700 [ "${show_progress:-0}" != "0" ] || exec >/dev/null
1701 # Note that this does not optimize _darcs/inventories/ :(
1702 darcs optimize || :
1706 esac
1709 # Create a matching .bndl header file for the all-in-one pack we just created
1710 # but only if we're not a fork (otherwise the bundle would not be complete)
1711 # and we are running at least Git version 1.7.2 (pack_is_complete always fails otherwise)
1712 if [ -n "$makebndl" ] && [ -n "$var_have_git_172" ]; then
1713 # There should only be one pack in $packs but do some checking...
1714 # The one we just created will have a .idx and will NOT have a .keep
1715 progress "~ [$proj] creating downloadable bundle header"
1716 pkbase=
1717 pkhead=
1718 IFS= read -r curhead <repack/HEAD.orig || :
1720 [ -s "objects/pack/pack-$packs.pack" ] &&
1721 [ -s "objects/pack/pack-$packs.idx" ] &&
1722 ! [ -e "objects/pack/pack-$packs.keep" ] &&
1723 pkhead="$(pack_is_complete "$PWD/objects/pack/pack-$packs.pack" \
1724 "$PWD/repack/packed-refs.orig" "$curhead")"
1725 then
1726 pkbase="objects/pack/pack-$packs"
1728 if [ -n "$pkbase" ] && [ -n "$pkhead" ]; then
1730 symref=
1731 case "$curhead" in "ref: refs/"?*|"ref:refs/"?*|"refs/"?*)
1732 symref="${curhead#ref:}"
1733 symref="${symref# }"
1734 esac
1735 bndlurl=
1736 [ -z "$cfg_httpbundleurl" ] || bndlurl=" url=$cfg_httpbundleurl/$proj.git/clone.bundle"
1737 echo "# v2 git bundle"
1738 LC_ALL=C sed -ne "/^$octet20$hexdig* refs\/[^ $tab]*\$/ p" <repack/packed-refs.orig
1739 if [ -n "$symref" ]; then
1740 printf "$pkhead HEAD\0symref=HEAD:%s%s\n" "$symref" "$bndlurl"
1741 else
1742 if [ -n "$bndlurl" ]; then
1743 printf "$pkhead HEAD\0%s\n" "${bndlurl# }"
1744 else
1745 echo "$pkhead HEAD"
1748 echo ""
1749 } >"$pkbase.bndl"
1750 bndletag="$("$cfg_basedir/bin/rangecgi" --etag -m 1 "$pkbase.bndl" "$pkbase.pack")" || :
1751 bndlsha="$(printf '%s' "$bndletag" | git hash-object --stdin)" || :
1752 if [ -n "$bndletag" ]; then
1753 case "$bndlsha" in $octet20*)
1754 bndlshatrailer="${bndlsha#????????}"
1755 bndlshaprefix="${bndlsha%$bndlshatrailer}"
1756 bndlname="$(TZ=UTC date +%Y%m%d_%H%M%S)-${bndlshaprefix:-0}"
1757 [ -d bundles ] || mkdir bundles
1758 echo "${pkbase#objects/pack/}.bndl" >"bundles/$bndlname"
1759 echo "${pkbase#objects/pack/}.pack" >>"bundles/$bndlname"
1760 ln -s -f -n "$bndlname" bundles/latest
1761 esac
1766 # Record the size of this repo as the sum of its clone packed-refs + *.pack sizes as 1024-byte blocks
1767 eval "reposizek=$(( $(
1768 echo 0 $(du -k repack/packed-refs.orig $(printf 'objects/pack/pack-%s.pack ' $packs) 2>/dev/null |
1769 LC_ALL=C awk '{print $1}') |
1770 LC_ALL=C sed -e 's/ / + /g') ))"
1771 config_set_raw girocco.reposizek "${reposizek:-0}"
1773 read -r rsighash <repack/packed-refs.sighash || :
1774 read -r repacksig <repack/repack.sighash || :
1776 # Now we're finally done with this
1777 rm -rf repack
1779 # Update various "signature" hashes used to help determine whether or
1780 # not it's okay to skip gc in the future (because there's nothing to do)
1782 psighash="$(if [ -n "$packs" ] || [ -n "$spacks" ]; then
1783 printf 'pack-%s.pack\n' $packs $spacks
1784 fi | LC_ALL=C sort | sig_hash_stdin)"
1785 config_set_raw girocco.gcsig.packs "$psighash"
1786 config_set_raw girocco.gcsig.refs "$rsighash"
1787 config_set_raw girocco.gcsig.repack "$repacksig"
1789 # We use $gcstart here to avoid a race where a push occurs during the gc itself
1790 # and the next future gc could be incorrectly skipped if we used the current
1791 # timestamp here instead
1792 config_set lastgc "$gcstart"
1793 rm -f "$lockf"
1795 progress "- [$proj] garbage check ($(date))"