From 2b0fe78d59c22901e6a8fd80c7ed18ab8715f2f2 Mon Sep 17 00:00:00 2001
From: "Kyle J. McKay" <mackyle@gmail.com>
Date: Wed, 10 Jun 2020 03:00:29 -0700
Subject: [PATCH] gc-util-functions.sh: combine loose packs as needed

Whenever pack_incremental_loose_objects is called, after
packing up the loose objects, if there are now too many
packs, attempt to combine them to reduce the total number
of packs.

Signed-off-by: Kyle J. McKay <mackyle@gmail.com>
---
 jobd/gc-util-functions.sh | 55 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/jobd/gc-util-functions.sh b/jobd/gc-util-functions.sh
index edb0eda..6eb73f3 100755
--- a/jobd/gc-util-functions.sh
+++ b/jobd/gc-util-functions.sh
@@ -87,6 +87,7 @@ lotsa_loose_objects() {
 # pack any existing, non-packed loose objects into a new _l.pack file then run prune-packed
 # note that prune-packed is NOT run beforehand -- the caller must do that if needed
 # loose objects need not be part of complete commits/trees as --weak-naming is used
+# if there end up being too many loose packs, attempt to combine the packs too
 pack_incremental_loose_objects() {
 	_lpacks="$(run_combine_packs </dev/null --names --loose --weak-naming --incremental --non-empty --all-progress-implied ${quiet:---progress} $packopts)"
 	if [ -n "$_lpacks" ]; then
@@ -96,4 +97,58 @@ pack_incremental_loose_objects() {
 		done
 		git prune-packed $quiet
 	fi
+	_packs=
+	_lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
+	_lpol="$_lpo --exclude-no-sfx _l"
+	{ _packs="$(list_packs --quiet --count $_lpol objects/pack || :)" || :; } 2>/dev/null
+	[ "${_packs:-0}" -lt 20 ] || {
+		combine_small_incremental_loose_packs
+		_packs=
+		{ _packs="$(list_packs --quiet --count $_lpol objects/pack || :)" || :; } 2>/dev/null
+		[ "${_packs:-0}" -lt 20 ] || combine_large_incremental_loose_packs
+	}
+}
+
+# combine small _l packs into larger pack(s) using --weak-naming
+# we avoid any non _l, keep, bndl or bitmap packs
+combine_small_incremental_loose_packs() {
+	_lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
+	_lpo="$_lpo --exclude-no-sfx _l"
+	_lpo="$_lpo --quiet --object-limit $var_redelta_threshold objects/pack"
+	while
+		_cnt="$(list_packs --count $_lpo)" || :
+		test "${_cnt:-0}" -ge 2
+	do
+		_newp="$(list_packs $_lpo | combine_packs_std --names --weak-naming --no-reuse-delta)"
+		# We need to identify these packs later so we don't combine_packs them
+		for _objpack in $_newp; do
+			rename_pack "objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" || :
+		done
+		v_cnt _newc $_newp
+		# be paranoid and exit the loop if we haven't reduced the number of packs
+		[ $_newc -lt $_cnt ] || break
+	done
+	return 0
+}
+
+# combine large[ish] _l packs into larger pack(s) using --weak-naming
+# we avoid any non _l, keep, bndl or bitmap packs
+combine_large_incremental_loose_packs() {
+	_lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
+	_lpo="$_lpo --exclude-no-sfx _l"
+	_lpo="$_lpo --quiet --exclude-limit -$(( ( $var_redelta_threshold / 2 ) + 1 )) objects/pack"
+	while
+		_cnt="$(list_packs --count $_lpo)" || :
+		test "${_cnt:-0}" -ge 2
+	do
+		_newp="$(list_packs $_lpo | combine_packs_std --names --weak-naming)"
+		# We need to identify these packs later so we don't combine_packs them
+		for _objpack in $_newp; do
+			rename_pack "objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" || :
+		done
+		v_cnt _newc $_newp
+		# be paranoid and exit the loop if we haven't reduced the number of packs
+		[ $_newc -lt $_cnt ] || break
+	done
+	return 0
 }
-- 
2.11.4.GIT