From 067e39d8c2d1951910e612880b5cbf34e8c78653 Mon Sep 17 00:00:00 2001 From: "Kyle J. McKay" Date: Wed, 11 Aug 2021 19:27:13 -0700 Subject: [PATCH] taskd/clone.sh: trim excessively large successful .clonelog files If a successful clone's .clonelog file exceeds 10000 lines, trim it back to the first 5000 lines plus a comment about how many lines have been omitted plus the last 5000 lines. Some very long running clones, especially git-svn clones, can produce multi-megabyte-sized .clonelog files that are not particularly useful, especially after a successful clone. Retaining the first and last 5000 lines of such a log file should more than suffice for a successful clone. For a failed clone, the .clonelog file remains untouched. Signed-off-by: Kyle J. McKay --- taskd/clone.sh | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/taskd/clone.sh b/taskd/clone.sh index 9745687..aba3cb6 100755 --- a/taskd/clone.sh +++ b/taskd/clone.sh @@ -795,5 +795,38 @@ EOT echo "Mirroring finished successfuly!" # In case this is a re-mirror, lastgc could have been set already so clear it now git config --unset gitweb.lastgc || : -rm .clone_in_progress + +# Finishing touches to .clonelog echo "$sizenote@OVER@" + +# We must now close the .clonelog file that is open on stdout and stderr +# otherwise it will not be possible to examine it and possibly trim it +exec >/dev/null 2>&1 || : + +# Trim an overly large successful clone log back down to something less wasteful +# Keep only 10000 lines (first 5000, last 5000) if there are more than 10003 lines +# This will limit the .clonelog file to roughly 800KB max (with 80 char average lines) +loglines=$(LC_ALL=C wc -l <.clonelog) || : +if [ "${loglines:-0}" -gt 10003 ]; then + # Try to be careful and never lose the .clonelog nor fail the + # clone at this point no matter what bizarre failures might occur + rm -f .clonelogtrim || : + if + test ! -e .clonelogtrim && + { >.clonelogtrim; } >/dev/null 2>&1 && + test -f .clonelogtrim + then + { + head -n 5000 .clonelog && + echo "" && + echo "[ ... elided $(( $loglines - 10000 )) middle lines ... ]" && + echo "" && + tail -n 5000 .clonelog + } >.clonelogtrim && + mv -f .clonelogtrim .clonelog || : + rm -f .clonelogtrim || : + fi +fi + +# Clone is no longer in progress +rm -f .clone_in_progress || : -- 2.11.4.GIT