From 18e1389dc8003194d32d3ede20b44736f8385157 Mon Sep 17 00:00:00 2001
From: "Kyle J. McKay" <mackyle@gmail.com>
Date: Fri, 12 Jun 2020 17:40:54 -0700
Subject: [PATCH] taskd/clone.sh: exponential backoff for git-svn retries

When `git-svn fetch` fails and makes no progress, double the
delay between attempts as long there is no progress made.

If any progress has been made, reset the delay back to
the minimum.

Increase the number of consecutive no-progress retries from
five to seven since the wait will be much longer between retries.

Since no retry can take place unless the server has been
successfully contacted and the initial branch layout set up,
any invalid `git-svn` clone source URL will still fail
immediately regardless of this change.

Additionally, since the update process (i.e. jobd/update.sh)
does *NOT* perform any delay+retry (rather it's left up to
the normal jobd.pl update retry mechanism), there's no impact
to jobd.pl operation with this change.

Signed-off-by: Kyle J. McKay <mackyle@gmail.com>
---
 taskd/clone.sh | 37 +++++++++++++++++++++++++------------
 1 file changed, 25 insertions(+), 12 deletions(-)

diff --git a/taskd/clone.sh b/taskd/clone.sh
index c6c84f8..296b696 100755
--- a/taskd/clone.sh
+++ b/taskd/clone.sh
@@ -362,10 +362,16 @@ case "$url" in
 			git hash-object -t blob --stdin )"' || :
 		}
 		svn_ret_err() { return "${1:-1}"; }
-		svn_retries=1000
+		svn_retries=1000 # maximum possible fetch attempts no matter what
+		svn_retry_backoff_start_half=60 # min retry wait is double this amount in seconds
+		svn_backoff_count=7 # max retry wait is $svn_retry_backoff_start_half * 2^$svn_backoff_count
+		# Cumulative backoff wait before giving up on consecutive no-progress retries
+		#   is approximately 2 * $svn_retry_backoff_start_half * 2^$svn_backoff_count
+		# For a $svn_backoff_count of 7 that works out to be exactly 4h14m
 		svn_progress=
 		v_get_svn_progress_fingerprint svn_progress
 		svn_progress_retries="$svn_retries"
+		svn_retry_backoff="$svn_retry_backoff_start_half"
 		svn_err=0
 		while [ "$svn_retries" -gt 0 ]; do
 			svn_retries="$(( $svn_retries - 1 ))"
@@ -390,33 +396,40 @@ case "$url" in
 			# Check to see if we made any progress
 			v_get_svn_progress_fingerprint svn_progress_now
 			if [ "$svn_progress_now" != "$svn_progress" ]; then
-				# we made progress, continue the loop
+				# we made progress, continue the loop with min wait
 				svn_progress="$svn_progress_now"
 				svn_progress_retries="$svn_retries"
+				svn_retry_backoff="$svn_retry_backoff_start_half"
 			else
 				# no progress, but we only give up after
-				# 5 no-progress attempts in a row
-				[ "$(( $svn_progress_retries - $svn_retries ))" -lt 5 ] || break # failure
+				# $svn_backoff_count no-progress attempts in a row
+				[ "$(( $svn_progress_retries - $svn_retries ))" -lt "$svn_backoff_count" ] ||
+					break # failure
+				# continue but only after twice the previous wait
+				# (which will still be the min wait if this is the
+				# first no-progress retry after making some progress)
 			fi
-			# Pause briefly before retrying to be friendly to the server
+			svn_retry_backoff="$(( 2 * $svn_retry_backoff ))"
+			# Pause for $svn_retry_backoff seconds before retrying to be friendly to the server
 			# Use that time to pack up loose objects if there are "lotsa" them
 			if ! lotsa_loose_objects_or_sopacks; then
-				echo 'Pausing for 120 seconds before retrying'
-				sleep 120
+				echo "Pausing for $svn_retry_backoff seconds before retrying ($(date))"
+				sleep "$svn_retry_backoff"
 			else
-				pausestop="$(( $(date '+%s') + 120 ))"
-				echo 'Pausing and packing loose objects for 120 seconds before retrying'
+				pausestop="$(( $(date '+%s') + $svn_retry_backoff ))"
+				echo "Pausing and packing loose objects for $svn_retry_backoff seconds before retrying ($(date))"
 				pack_incremental_loose_objects_if_lockable ||
-				echo "Packing skipped: $lockerr"
+				echo "Packing skipped (only pausing): $lockerr"
 				timenow="$(date '+%s')"
 				if [ "$timenow" -lt "$pausestop" ]; then
 					sleepamt="$(( $pausestop - $timenow ))"
-					[ "$sleepamt" -le 120 ] || sleepamt=120 # paranoia check
+					[ "$sleepamt" -le "$svn_retry_backoff" ] ||
+						sleepamt="$svn_retry_backoff" # paranoia check
 					sleep "$sleepamt"
 				fi
 			fi
 			cleanup_git_svn_leftovers
-			echo 'Retrying fetch'
+			echo "Retrying fetch ($(date))"
 		done
 		[ "${svn_err:-1}" -eq 0 ] || svn_ret_err "$svn_err"
 		test ${svn_err:-1} -eq 0
-- 
2.11.4.GIT